Tags should be unescaped before storing

Test tag is Test tag

5 files changed, 37 insertions, 32 deletions

diff --git a/src/main/java/com/juick/server/CommandsManager.java b/src/main/java/com/juick/server/CommandsManager.java
index f6f29941..fdea0d83 100644
--- a/src/main/java/com/juick/server/CommandsManager.java
+++ b/src/main/java/com/juick/server/CommandsManager.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.commons.lang3.reflect.MethodUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.text.StringEscapeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -91,7 +92,7 @@ public class CommandsManager {
         if (strippedData.startsWith("?OTR")) {
             return CommandResult.fromString("?OTR Error: we are not using OTR");
         }
-        String input = MessageUtils.stripNonSafeUrls(strippedData);
+        String input = StringEscapeUtils.unescapeHtml4(MessageUtils.stripNonSafeUrls(strippedData));
         Optional<Method> cmd = MethodUtils.getMethodsListWithAnnotation(getClass(), UserCommand.class).stream()
                 .filter(m -> Pattern.compile(m.getAnnotation(UserCommand.class).pattern(),
                         m.getAnnotation(UserCommand.class).patternFlags()).matcher(input).matches())
diff --git a/src/main/java/com/juick/service/MessagesServiceImpl.java b/src/main/java/com/juick/service/MessagesServiceImpl.java
index 3e09d204..2bae04e6 100644
--- a/src/main/java/com/juick/service/MessagesServiceImpl.java
+++ b/src/main/java/com/juick/service/MessagesServiceImpl.java
@@ -765,7 +765,7 @@ public class MessagesServiceImpl extends BaseJdbcService implements MessagesServ
 
         return getNamedParameterJdbcTemplate().queryForList(
                 "SELECT messages.message_id FROM messages_tags INNER JOIN messages " +
-                        " USING (message_id) WHERE messages.user_id = :uid AND messages_tags.tag_id = :tid " +
+                        " ON messages.message_id = messages_tags.message_id WHERE messages.user_id = :uid AND messages_tags.tag_id = :tid " +
                         (before > 0 ?
                                 " AND messages.message_id < :before " : StringUtils.EMPTY) +
                         " AND messages.privacy >= :privacy ORDER BY messages.message_id DESC LIMIT 20",
diff --git a/src/main/resources/templates/views/macros/tags.html b/src/main/resources/templates/views/macros/tags.html
index defed8e6..08687f5a 100644
--- a/src/main/resources/templates/views/macros/tags.html
+++ b/src/main/resources/templates/views/macros/tags.html
@@ -1,11 +1,11 @@
 {% macro tags(uname="", tagsList) %}
 {% for tag in tagsList %}
-<a href="/{{ uname }}/?tag={{ tag | urlencode }}">{{ tag | raw }}</a>
+<a href="/{{ uname }}/?tag={{ tag | urlencode }}">{{ tag }}</a>
 {% endfor %}
 {% endmacro %}
 
 {% macro allTags(baseUri, tagsList) %}
 {% for tag in tagsList %}
-<a href="{{ baseUri }}tag/{{ tag | urlencode }}">#{{ tag | raw }}</a>
+<a href="{{ baseUri }}tag/{{ tag | urlencode }}">#{{ tag }}</a>
 {% endfor %}
 {% endmacro %}
\ No newline at end of file
diff --git a/src/main/resources/templates/views/partial/tags.html b/src/main/resources/templates/views/partial/tags.html
index 3235213e..4d05b7fb 100644
--- a/src/main/resources/templates/views/partial/tags.html
+++ b/src/main/resources/templates/views/partial/tags.html
@@ -1,3 +1,3 @@
 {% for tag in tags %}
-    <a href="/tag/{{ tag | urlencode }}" title="{{ tag }}">{{ tag | raw }}</a>
+    <a href="/tag/{{ tag | urlencode }}" title="{{ tag }}">{{ tag }}</a>
 {% endfor %}
\ No newline at end of file
diff --git a/src/test/java/com/juick/server/tests/ServerTests.java b/src/test/java/com/juick/server/tests/ServerTests.java
index afe7f659..65e19b89 100644
--- a/src/test/java/com/juick/server/tests/ServerTests.java
+++ b/src/test/java/com/juick/server/tests/ServerTests.java
@@ -1028,22 +1028,32 @@ public class ServerTests {
         assertThat(tags.get(0).getName(), equalTo("yo"));
     }
     @Test
-    public void messageParserSerializer() throws ParserConfigurationException,
-            IOException, SAXException, JAXBException {
-        Set<Tag> tags = MessageUtils.parseTags("test test" + (char) 0xA0 + "2 test3");
-        List<Tag> tagList = new ArrayList<>(tags);
+    public void messageParserSerializer() throws Exception {
+        String tagsString = "test test" + (char) 0xA0 + "2 test 3";
+        Set<Tag> tags = MessageUtils.parseTags(tagsString);
+        List<Tag> tagList = tags.stream().map(t -> tagService.getTag(t.getName(), true))
+                .collect(Collectors.toList());
         assertEquals("First tag must be", "test", tagList.get(0).getName());
-        assertEquals("Third tag must be", "test3", tagList.get(2).getName());
+        assertEquals("Third tag must be", "test 3", tagList.get(2).getName());
         assertEquals("Count of tags must be", 3, tagList.size());
-        Message msg = new Message();
-        msg.setTags(tags);
-        Instant currentDate = Instant.now();
-        msg.setCreated(currentDate);
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
+        MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
+        HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
+
+        map.add("body", "*test *test&nbsp;2 *test 3 YO");
+        map.add("hash", userService.getHashByUID(ugnich.getUid()));
+        ResponseEntity<CommandResult> result = restTemplate.postForEntity(
+                "/api/post",
+                request, CommandResult.class);
+        assertThat(result.getStatusCode(), is(HttpStatus.OK));
+        Message msg = result.getBody().getNewMessage().orElseThrow();
+        Instant currentDate = msg.getCreated();
         String jsonMessage = jsonMapper.writeValueAsString(msg);
-        assertEquals("date should be in timestamp field", DateFormattersHolder.getMessageFormatterInstance().format(currentDate),
+        assertEquals("date should be in timestamp field",
+                DateFormattersHolder.getMessageFormatterInstance().format(currentDate),
                 JsonPath.read(jsonMessage, "$.timestamp"));
 
-
         JAXBContext context = JAXBContext
                 .newInstance(Message.class);
         Marshaller m = context.createMarshaller();
@@ -1056,22 +1066,16 @@ public class ServerTests {
         Document doc = db.parse(new ByteArrayInputStream(sw.toString().getBytes(StandardCharsets.UTF_8)));
         Node juickNode = doc.getDocumentElement();
         NamedNodeMap attrs = juickNode.getAttributes();
-        assertEquals("date should be in ts field", DateFormattersHolder.getMessageFormatterInstance().format(currentDate),
+        assertEquals("date should be in ts field",
+                DateFormattersHolder.getMessageFormatterInstance().format(currentDate),
                 attrs.getNamedItem("ts").getNodeValue());
-    }
-    @Test
-    public void restTemplateTests() {
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
-        MultiValueMap<String, String> map= new LinkedMultiValueMap<>();
-        HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
 
-        map.add("body", "yo");
-        map.add("hash", userService.getHashByUID(ugnich.getUid()));
-        ResponseEntity<CommandResult> result = restTemplate.postForEntity(
-                    "/api/post",
-                    request, CommandResult.class);
-        assertThat(result.getStatusCode(), is(HttpStatus.OK));
+        MvcResult apiResult = mockMvc.perform(get("/api/thread?mid=" + msg.getMid()))
+                .andExpect(status().isOk())
+                .andReturn();
+        List<Message> fromApi = jsonMapper.readValue(apiResult.getResponse().getContentAsString(),
+                new TypeReference<List<Message>>() {});
+        assertThat(fromApi.get(0).getTags(), is(tags));
     }
     @Test
     public void emptyAuthenticatedPostShouldThrowBadRequest() throws Exception {
@@ -1486,9 +1490,9 @@ public class ServerTests {
         Writer writer = new StringWriter();
         template.evaluate(writer,
                 Collections.singletonMap("tagsList",
-                        Collections.singletonList(StringEscapeUtils.escapeHtml4(new Tag(">_<").getName()))));
+                        Collections.singletonList(new Tag(">_<").getName())));
         String output = writer.toString().trim();
-        assertThat(output, equalTo("<a href=\"/ugnich/?tag=%26gt%3B_%26lt%3B\">&gt;_&lt;</a>"));
+        assertThat(output, equalTo("<a href=\"/ugnich/?tag=%3E_%3C\">&gt;_&lt;</a>"));
     }
 
     public DomElement fetchMeta(String url, String name) throws IOException {