sape: switch to xml output

8 files changed, 188 insertions, 67 deletions

diff --git a/src/main/java/com/juick/www/ad/SapeService.java b/src/main/java/com/juick/www/ad/SapeService.java
index 4ef4a213..3c35f320 100644
--- a/src/main/java/com/juick/www/ad/SapeService.java
+++ b/src/main/java/com/juick/www/ad/SapeService.java
@@ -60,8 +60,8 @@ public class SapeService {
                 && visitor.isAnonymous();
         model.addAttribute("showAdv", showAdv);
         if (showAdv) {
-            String links = sape.getPageLinks(requestURI, sapeCookie).render();
-            model.addAttribute("links", links);
+            sape.getPageLinks(requestURI, sapeCookie)
+                    .ifPresent(sapePageLinks -> model.addAttribute("links", sapePageLinks.render()));
         }
     }
 }
diff --git a/src/main/java/com/juick/www/ad/models/Page.java b/src/main/java/com/juick/www/ad/models/Page.java
new file mode 100644
index 00000000..d5bb8ae3
--- /dev/null
+++ b/src/main/java/com/juick/www/ad/models/Page.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2008-2020, Juick
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.juick.www.ad.models;
+
+import java.util.List;
+
+public class Page {
+    private final String uri;
+    private final List<String> links;
+
+    public Page(String uri, List<String> links) {
+        this.uri = uri;
+        this.links = links;
+    }
+
+    public String getUri() {
+        return uri;
+    }
+    public List<String> getLinks() {
+        return links;
+    }
+}
diff --git a/src/main/java/com/juick/www/ad/models/Site.java b/src/main/java/com/juick/www/ad/models/Site.java
new file mode 100644
index 00000000..fbe1c033
--- /dev/null
+++ b/src/main/java/com/juick/www/ad/models/Site.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2008-2020, Juick
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.juick.www.ad.models;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Site {
+    private final String siteUrl;
+    private final String delimiter;
+    private final List<Page> pages;
+    private final String code;
+
+    public Site(String siteUrl, String code, String delimiter, List<Page> pages) {
+        this.siteUrl = siteUrl;
+        this.code = code;
+        this.delimiter = delimiter;
+        this.pages = pages;
+    }
+
+    public String getSiteUrl() {
+        return siteUrl;
+    }
+    public String getDelimiter() {
+        return delimiter;
+    }
+    public List<Page> getPages() {
+        return pages;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public static Site fromXMLData(String data) throws ParserConfigurationException, IOException, SAXException {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        DocumentBuilder builder = factory.newDocumentBuilder();
+        Document document = builder.parse(IOUtils.toInputStream(data, StandardCharsets.UTF_8));
+        Element sape = document.getDocumentElement();
+        String code = StringUtils.EMPTY;
+        String siteUrl = sape.getAttribute("site_url");
+        String delimiter = sape.getAttribute("delimiter");
+        NodeList pageNodes = sape.getElementsByTagName("page");
+        List<Page> pages = new ArrayList<>(pageNodes.getLength());
+        for (int i = 0; i < pageNodes.getLength(); i++) {
+            Element pageElement = (Element) pageNodes.item(i);
+            String uri = pageElement.getAttribute("uri");
+            if (uri.equals("*")) {
+                code = pageElement.getTextContent();
+            }
+            NodeList linkNodes = pageElement.getElementsByTagName("link");
+            List<String> links = new ArrayList<>(linkNodes.getLength());
+            for (int j = 0; j < linkNodes.getLength(); j++) {
+                links.add(linkNodes.item(j).getTextContent());
+            }
+            pages.add(new Page(uri, links));
+        }
+        return new Site(siteUrl, code, delimiter, pages);
+    }
+}
diff --git a/src/main/java/ru/sape/Sape.java b/src/main/java/ru/sape/Sape.java
index a94bcc62..6b28be25 100644
--- a/src/main/java/ru/sape/Sape.java
+++ b/src/main/java/ru/sape/Sape.java
@@ -4,6 +4,7 @@
 package ru.sape;
 
 import java.net.URI;
+import java.util.Optional;
 
 public class Sape {
 
@@ -19,7 +20,8 @@ public class Sape {
     }
     public boolean debug = false;
 
-    public SapePageLinks getPageLinks(URI requestUri, String cookie) {
-        return new SapePageLinks(sapePageLinkConnection, sapeUser, requestUri, cookie, debug);
+    public Optional<SapePageLinks> getPageLinks(URI requestUri, String cookie) {
+        return sapePageLinkConnection.getData()
+                .map(site -> new SapePageLinks(site, sapeUser, requestUri, cookie, debug));
     }
 }
diff --git a/src/main/java/ru/sape/SapeConnection.java b/src/main/java/ru/sape/SapeConnection.java
index ee5a5e5c..cec308bf 100644
--- a/src/main/java/ru/sape/SapeConnection.java
+++ b/src/main/java/ru/sape/SapeConnection.java
@@ -1,8 +1,8 @@
 package ru.sape;
 
-import com.github.ooxi.phparser.SerializedPhpParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import com.juick.www.ad.models.Site;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -66,17 +66,16 @@ public class SapeConnection {
             }
         }
     }
-    Map<String, Object> cached;
+    Site cachedSite;
     long cacheUpdated;
 
-    @SuppressWarnings("unchecked")
-    public Map<String, Object> getData() {
+    public Optional<Site> getData() {
         if (cacheLifeTime <= (System.currentTimeMillis() - cacheUpdated) / 1000) {
             for (String server : serverList) {
                 String data;
 
                 try {
-                    data = fetchRemoteFile(server, dispenserPath + "&charset=UTF-8");
+                    data = fetchRemoteFile(server, dispenserPath + "&charset=UTF-8&as_xml=true");
                 } catch (IOException e1) {
                     continue;
                 }
@@ -88,7 +87,7 @@ public class SapeConnection {
                 }
 
                 try {
-                    cached = (Map<String, Object>) new SerializedPhpParser(data).parse();
+                    cachedSite = Site.fromXMLData(data);
                 } catch (Exception e) {
                     logger.error("Can't parse Sape data", e);
                     continue;
@@ -96,14 +95,14 @@ public class SapeConnection {
 
                 cacheUpdated = System.currentTimeMillis();
 
-                return cached;
+                return Optional.of(cachedSite);
             }
 
             logger.error("Unable to fetch Sape data");
 
-            return Collections.emptyMap();
+            return Optional.empty();
         }
 
-        return cached;
+        return Optional.of(cachedSite);
     }
 }
diff --git a/src/main/java/ru/sape/SapePageLinks.java b/src/main/java/ru/sape/SapePageLinks.java
index 77715aea..52def3f6 100644
--- a/src/main/java/ru/sape/SapePageLinks.java
+++ b/src/main/java/ru/sape/SapePageLinks.java
@@ -1,71 +1,39 @@
 package ru.sape;
 
+import com.juick.www.ad.models.Site;
 import org.apache.commons.lang3.StringUtils;
 
 import java.net.URI;
-import java.util.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public class SapePageLinks {
 
-    private boolean showCode;
+    private final boolean showCode;
+    private final String linkDelimiter;
+    private final List<String> pageLinks;
 
-    @SuppressWarnings("unchecked")
-    public SapePageLinks(SapeConnection sapeConnection, String sapeUser, URI request, String sapeCookie, boolean showCode) {
+    public SapePageLinks(Site site, String sapeUser, URI request, String sapeCookie, boolean showPageCode) {
         String req = StringUtils.isNotEmpty(request.getQuery()) ? request.getPath() + "?" + request.getQuery()
                 : request.getPath();
-        if (sapeUser.equals(sapeCookie)) {
-            showCode = true;
-        }
-
-        Map<String, Object> data = sapeConnection.getData();
 
-        if (data.containsKey("__sape_delimiter__")) {
-            linkDelimiter = (String) data.get("__sape_delimiter__");
-        }
-
-        if (data.containsKey(req)) {
-            pageLinks = new ArrayList<>(((Map<Object, String>) data.get(req)).values());
-        }
+        linkDelimiter = site.getDelimiter();
 
-        if (data.containsKey("__sape_new_url__")) {
-            if (showCode) {
-                Object newUrl = data.get("__sape_new_url__");
+        this.showCode = showPageCode || sapeUser.equals(sapeCookie);
 
-                if (newUrl instanceof Map) {
-                    pageLinks = new ArrayList<>(((Map<Object, String>) newUrl).values());
-                } else {
-                    pageLinks = new ArrayList<>(Collections.singletonList((String) newUrl));
-                }
-            }
+        if (showCode) {
+            pageLinks = Collections.singletonList(site.getCode());
+        } else {
+            pageLinks = site.getPages().stream().filter(page -> page.getUri().equals(req))
+                    .flatMap(page -> page.getLinks().stream()).collect(Collectors.toList());
         }
-
-        this.showCode = showCode;
     }
-    private String linkDelimiter = ".";
-    private List<String> pageLinks = new ArrayList<>();
 
-    public String render() {
-        return render(-1);
-    }
 
-    public String render(int count) {
+    public String render() {
         StringBuilder s = new StringBuilder();
-
-        if (count < 0) {
-            count = pageLinks.size();
-        }
-
-        for (Iterator<String> i = pageLinks.iterator(); i.hasNext() && count > 0; count--) {
-            if (s.length() > 0) {
-                s.append(linkDelimiter);
-            }
-
-            String l = i.next();
-
-            s.append(l);
-
-            i.remove();
-        }
+        s.append(String.join(linkDelimiter, pageLinks));
 
         if (showCode) {
             s.insert(0, "<sape_noindex>");
diff --git a/src/test/java/com/juick/server/tests/ServerTests.java b/src/test/java/com/juick/server/tests/ServerTests.java
index 43a0dc7b..0bc23bf7 100644
--- a/src/test/java/com/juick/server/tests/ServerTests.java
+++ b/src/test/java/com/juick/server/tests/ServerTests.java
@@ -74,7 +74,6 @@ import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
@@ -105,14 +104,14 @@ import org.springframework.web.client.ResourceAccessException;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponents;
 import org.springframework.web.util.UriComponentsBuilder;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
+import org.w3c.dom.*;
+import org.xml.sax.SAXException;
 import rocks.xmpp.addr.Jid;
 import rocks.xmpp.core.session.Extension;
 import rocks.xmpp.core.session.XmppSession;
 import rocks.xmpp.core.session.XmppSessionConfiguration;
+import com.juick.www.ad.models.Site;
+import ru.sape.SapePageLinks;
 
 import javax.inject.Inject;
 import javax.servlet.http.Cookie;
@@ -122,6 +121,7 @@ import javax.xml.bind.Marshaller;
 import javax.xml.bind.Unmarshaller;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 import java.io.*;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
@@ -246,6 +246,8 @@ public class ServerTests {
     private Resource jpegNoJfifTiff;
     @Value("classpath:Transparent.gif")
     private Resource invisiblePixel;
+    @Value("classpath:sape.xml")
+    private Resource sapeOutput;
     @Inject
     AppleClientSecretGenerator clientSecretGenerator;
     @Inject
@@ -2420,4 +2422,18 @@ public class ServerTests {
                 .andExpect(status().is3xxRedirection())
                 .andExpect(redirectedUrl("http://localhost/login"));
     }
+    @Test
+    public void linksTest() throws IOException, ParserConfigurationException, SAXException {
+
+        Site site = Site.fromXMLData(IOUtils.toString(sapeOutput.getInputStream(), StandardCharsets.UTF_8));
+        assertThat(site.getPages().size(), is(3));
+        assertThat(site.getPages().get(0).getLinks().size(), is(2));
+        assertThat(site.getCode(), is("<!--1234567890123-->"));
+        SapePageLinks botLinks = new SapePageLinks(site, "ugnich", URI.create("http://localhost/"), "ugnich", false);
+        assertThat(botLinks.render(), is("<sape_noindex><!--1234567890123--></sape_noindex>"));
+        SapePageLinks visitorLinks = new SapePageLinks(site, "ugnich", URI.create("http://localhost/"), null, false);
+        assertThat(visitorLinks.render(), is(" Тест <a href=\"https://example.com\" target=\"_blank\">ссылки</a> - passed. .  Тест <a href=\"https://example.com\" target=\"_blank\">ссылки 2</a> - passed. "));
+        SapePageLinks emptyLinks = new SapePageLinks(site, "ugnich", URI.create("http://localhost/yo"), null, false);
+        assertThat(emptyLinks.render(), is(emptyString()));
+    }
 }
diff --git a/src/test/resources/sape.xml b/src/test/resources/sape.xml
new file mode 100644
index 00000000..d1245ed4
--- /dev/null
+++ b/src/test/resources/sape.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<sape site_url="http://localhost" delimiter=". ">
+  <page uri="/">
+    <link><![CDATA[ Тест <a href="https://example.com" target="_blank">ссылки</a> - passed. ]]></link>
+    <link><![CDATA[ Тест <a href="https://example.com" target="_blank">ссылки 2</a> - passed. ]]></link>
+  </page>
+  <page uri="/ugnich/">
+    <link><![CDATA[ Тест <a href="https://ugnich.com" target="_blank">ссылки</a> - passed. ]]></link>
+  </page>
+  <page uri="*"><![CDATA[<!--1234567890123-->]]></page>
+  <sape_page_obligatory_output><![CDATA[]]></sape_page_obligatory_output>
+</sape>
\ No newline at end of file