1.9.10/src/test/org/codehaus/jackson/smile/TestSmileParserSymbolHandling.java

package org.codehaus.jackson.smile;

import java.io.*;
import java.util.*;

import org.codehaus.jackson.*;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.sym.BytesToNameCanonicalizer;

/**
 * Unit tests for verifying that symbol handling works as planned, including
 * efficient reuse of names encountered during parsing.
 */
public class TestSmileParserSymbolHandling
	extends SmileTestBase
{
    /*
    /**********************************************************
    /* Helper types, constants
    /**********************************************************
     */
    
    private final static String[] SHARED_SYMBOLS = new String[] {
            "g", "J", "v", "B", "S", "JAVA",
            "h", "J", "LARGE", 
            "JAVA", "J", "SMALL"
    };

    static class MediaItem
    {
        public Content content;
        public Image[] images;
    }

    public enum Size { SMALL, LARGE; }
    public enum Player { JAVA, FLASH; }
    
    static class Image
    {
        public String uri;
        public String title;
        public int width;
        public int height;
        public Size size;

        public Image() { }
        public Image(String uri, String title, int w, int h, Size s)
        {
            this.uri = uri;
            this.title = title;
            width = w;
            height = h;
            size = s;
        }
    }

    static class Content
    {
        public Player player;
        public String uri;
        public String title;
        public int width;
        public int height;
        public String format;
        public long duration;
        public long size;
        public int bitrate;
        public String[] persons;
        public String copyright;
    }
        
    /*
    /**********************************************************
    /* Unit tests
    /**********************************************************
     */

    public void testSimple() throws IOException
    {
        final String STR1 = "a";
		
    	byte[] data = _smileDoc("{ "+quote(STR1)+":1, \"foobar\":2, \"longername\":3 }");
    	SmileFactory f = new SmileFactory();
    	SmileParser p = _smileParser(f, data);
    	final BytesToNameCanonicalizer symbols1 = p._symbols;
    	assertEquals(0, symbols1.size());
    	
    	assertEquals(JsonToken.START_OBJECT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	// field names are interned:
    	assertSame(STR1, p.getCurrentName());
    	assertEquals(1, symbols1.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	assertSame("foobar", p.getCurrentName());
    	assertEquals(2, symbols1.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	assertSame("longername", p.getCurrentName());
    	assertEquals(3, symbols1.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.END_OBJECT, p.nextToken());
    	assertNull(p.nextToken());
    	assertEquals(3, symbols1.size());
        p.close();

        // but let's verify that symbol table gets reused properly
    	p = _smileParser(f, data);
    	BytesToNameCanonicalizer symbols2 = p._symbols;
    	// symbol tables are not reused, but contents are:
    	assertNotSame(symbols1, symbols2);
    	assertEquals(3, symbols2.size());

    	assertEquals(JsonToken.START_OBJECT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	// field names are interned:
    	assertSame(STR1, p.getCurrentName());
    	assertEquals(3, symbols2.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	assertSame("foobar", p.getCurrentName());
    	assertEquals(3, symbols2.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.FIELD_NAME, p.nextToken());
    	assertSame("longername", p.getCurrentName());
    	assertEquals(3, symbols2.size());
    	assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
    	assertEquals(JsonToken.END_OBJECT, p.nextToken());
    	assertNull(p.nextToken());
    	assertEquals(3, symbols2.size());
        p.close();

        assertEquals(3, symbols2.size());
    	p.close();
    }

    public void testSharedNames() throws IOException
    {
        final int COUNT = 19000;
        
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.WRITE_HEADER, false);
        f.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, true);
        ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
        JsonGenerator gen = f.createJsonGenerator(out);
        gen.writeStartArray();
        Random rnd = new Random(COUNT);
        for (int i = 0; i < COUNT; ++i) {
            gen.writeStartObject();
            int nr = rnd.nextInt() % 1200;
            gen.writeNumberField("f"+nr, nr);
            gen.writeEndObject();
        }
        gen.writeEndArray();
        gen.close();
        byte[] json = out.toByteArray();

        // And verify 
        f.configure(SmileParser.Feature.REQUIRE_HEADER, false);
        JsonParser jp = f.createJsonParser(json);
        assertToken(JsonToken.START_ARRAY, jp.nextToken());
        rnd = new Random(COUNT);
        for (int i = 0; i < COUNT; ++i) {
            assertToken(JsonToken.START_OBJECT, jp.nextToken());
            int nr = rnd.nextInt() % 1200;
            String name = "f"+nr;
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals(name, jp.getCurrentName());
            assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
            assertEquals(nr, jp.getIntValue());
            assertToken(JsonToken.END_OBJECT, jp.nextToken());
            
        }
        assertToken(JsonToken.END_ARRAY, jp.nextToken());
    }

    public void testSharedStrings() throws IOException
    {
        final int count = 19000;
        byte[] baseline = writeStringValues(false, count);
        assertEquals(763589, baseline.length);
        verifyStringValues(baseline, count);
        
        // and then shared; should be much smaller
        byte[] shared = writeStringValues(true, count);
        if (shared.length >= baseline.length) {
            fail("Expected shared String length < "+baseline.length+", was "+shared.length);
        }
        verifyStringValues(shared, count);
    }

    public void testSharedStringsInArrays() throws IOException
    {
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);

        ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
        JsonGenerator gen = f.createJsonGenerator(out);
        gen.writeStartArray();
        for (String value : SHARED_SYMBOLS) {
            gen.writeString(value);
        }
        gen.writeEndArray();
        gen.close();
        
        byte[] smile = out.toByteArray();

        JsonParser jp = f.createJsonParser(smile);
        assertToken(JsonToken.START_ARRAY, jp.nextToken());
        for (String value : SHARED_SYMBOLS) {
            assertToken(JsonToken.VALUE_STRING, jp.nextToken());
            assertEquals(value, jp.getText());
        }
        assertToken(JsonToken.END_ARRAY, jp.nextToken());
    }

    public void testSharedStringsInObject() throws IOException
    {
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
        ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
        JsonGenerator gen = f.createJsonGenerator(out);
        gen.writeStartObject();
        for (int i = 0; i < SHARED_SYMBOLS.length; ++i) {
            gen.writeFieldName("a"+i);
            gen.writeString(SHARED_SYMBOLS[i]);
        }
        gen.writeEndObject();
        gen.close();
        
        byte[] smile = out.toByteArray();

        JsonParser jp = f.createJsonParser(smile);
        assertToken(JsonToken.START_OBJECT, jp.nextToken());
        for (int i = 0; i < SHARED_SYMBOLS.length; ++i) {
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals("a"+i, jp.getCurrentName());
            assertToken(JsonToken.VALUE_STRING, jp.nextToken());
            assertEquals(SHARED_SYMBOLS[i], jp.getText());
        }
        assertToken(JsonToken.END_OBJECT, jp.nextToken());
    }

    public void testSharedStringsMixed() throws IOException
    {
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
        
        ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
        JsonGenerator gen = f.createJsonGenerator(out);
        gen.writeStartObject();

        gen.writeFieldName("media");
        gen.writeStartObject();

        gen.writeStringField("uri", "g");
        gen.writeStringField("title", "J");
        gen.writeNumberField("width", 640);
        gen.writeStringField("format", "v");
        gen.writeFieldName("persons");
        gen.writeStartArray();
        gen.writeString("B");
        gen.writeString("S");
        gen.writeEndArray();
        gen.writeStringField("player", "JAVA");
        gen.writeStringField("copyright", "NONE");

        gen.writeEndObject(); // media

        gen.writeFieldName("images");
        gen.writeStartArray();

        // 3 instances of identical entries
        for (int i = 0; i < 3; ++i) {
            gen.writeStartObject();
            gen.writeStringField("uri", "h");
            gen.writeStringField("title", "J");
            gen.writeNumberField("width", 1024);
            gen.writeNumberField("height", 768);
            gen.writeEndObject();
        }
        gen.writeEndArray();
        
        gen.writeEndObject();
        gen.close();
        
        byte[] smile = out.toByteArray();

        JsonParser jp = f.createJsonParser(smile);
        assertToken(JsonToken.START_OBJECT, jp.nextToken());

        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("media", jp.getCurrentName());

        assertToken(JsonToken.START_OBJECT, jp.nextToken());
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("uri", jp.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("g", jp.getText());
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("title", jp.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("J", jp.getText());
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("width", jp.getCurrentName());
        assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
        assertEquals(640, jp.getIntValue());
        
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("format", jp.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("v", jp.getText());
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("persons", jp.getCurrentName());
        assertToken(JsonToken.START_ARRAY, jp.nextToken());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("B", jp.getText());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("S", jp.getText());
        assertToken(JsonToken.END_ARRAY, jp.nextToken());

        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("player", jp.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("JAVA", jp.getText());
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("copyright", jp.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, jp.nextToken());
        assertEquals("NONE", jp.getText());
        
        assertToken(JsonToken.END_OBJECT, jp.nextToken()); // media
        
        assertToken(JsonToken.FIELD_NAME, jp.nextToken());
        assertEquals("images", jp.getCurrentName());
        assertToken(JsonToken.START_ARRAY, jp.nextToken());

        // 3 instances of identical entries:
        for (int i = 0; i < 3; ++i) {
            assertToken(JsonToken.START_OBJECT, jp.nextToken());
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals("uri", jp.getCurrentName());
            assertToken(JsonToken.VALUE_STRING, jp.nextToken());
            assertEquals("h", jp.getText());
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals("title", jp.getCurrentName());
            assertToken(JsonToken.VALUE_STRING, jp.nextToken());
            assertEquals("J", jp.getText());
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals("width", jp.getCurrentName());
            assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
            assertEquals(1024, jp.getIntValue());
            assertToken(JsonToken.FIELD_NAME, jp.nextToken());
            assertEquals("height", jp.getCurrentName());
            assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
            assertEquals(768, jp.getIntValue());
            assertToken(JsonToken.END_OBJECT, jp.nextToken());
        }
        
        assertToken(JsonToken.END_ARRAY, jp.nextToken()); // images
        
        assertToken(JsonToken.END_OBJECT, jp.nextToken());
    }

    public void testDataBindingAndShared() throws IOException
    {
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
        MediaItem item = new MediaItem();
        Content c = new Content();
        c.uri = "g";
        c.title = "J";
        c.width = 640;
        c.height = 480;
        c.format = "v";
        c.duration = 18000000L;
        c.size = 58982400L;
        c.bitrate = 262144;
        c.persons = new String[] { "B", "S" };
        c.player = Player.JAVA;
        c.copyright = "NONE";
        item.content = c;
        item.images = new Image[] {
            new Image("h", "J", 1024, 768, Size.LARGE),
            new Image("h", "J", 320, 240, Size.LARGE)
        };

        // Ok: let's just do quick comparison (yes/no)...
        ObjectMapper plain = new ObjectMapper();
        ObjectMapper smiley = new ObjectMapper(f);
        String exp = plain.writeValueAsString(item);
        byte[] smile = smiley.writeValueAsBytes(item);
        MediaItem result = smiley.readValue(smile, 0, smile.length, MediaItem.class);
        String actual = plain.writeValueAsString(result);
        assertEquals(exp, actual);
    }

    /**
     * Reproducing [JACKSON-561] (and [JACKSON-562])
     */
    public void testIssue562() throws IOException
    {
        JsonFactory factory = new SmileFactory();
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        JsonGenerator gen = factory.createJsonGenerator(bos);
        gen.writeStartObject();
        gen.writeFieldName("z_aaaabbbbccccddddee");
        gen.writeString("end");
        gen.writeFieldName("a_aaaabbbbccccddddee");
        gen.writeString("start");
        gen.writeEndObject();
        gen.close();

        JsonParser parser = factory.createJsonParser(bos.toByteArray());
        assertToken(JsonToken.START_OBJECT, parser.nextToken());

        assertToken(JsonToken.FIELD_NAME, parser.nextToken());
        assertEquals("z_aaaabbbbccccddddee", parser.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, parser.nextToken());
        assertEquals("end", parser.getText());

        // This one fails...
        assertToken(JsonToken.FIELD_NAME, parser.nextToken());
        assertEquals("a_aaaabbbbccccddddee", parser.getCurrentName());
        assertToken(JsonToken.VALUE_STRING, parser.nextToken());
        assertEquals("start", parser.getText());

        assertToken(JsonToken.END_OBJECT, parser.nextToken());
    }

    /**
     * Verification that [JACKSON-564] was fixed.
     */
    public void testIssue564() throws Exception
    {
        JsonFactory factory = new SmileFactory();

        ByteArrayOutputStream bos1 = new ByteArrayOutputStream();
        JsonGenerator generator = factory.createJsonGenerator(bos1);
        generator.writeStartObject();
        generator.writeFieldName("query");
        generator.writeStartObject();
        generator.writeFieldName("term");
        generator.writeStartObject();
        generator.writeStringField("doc.payload.test_record_main.string_not_analyzed__s", "foo");
        generator.writeEndObject();
        generator.writeEndObject();
        generator.writeEndObject();
        generator.close();

        JsonParser parser = factory.createJsonParser(bos1.toByteArray());
        JsonToken token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        assertEquals("query", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        assertEquals("term", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        assertEquals("doc.payload.test_record_main.string_not_analyzed__s", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.VALUE_STRING, token);
        assertEquals("foo", parser.getText());
        parser.close();

        ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
        generator = factory.createJsonGenerator(bos2);
        generator.writeStartObject();
        generator.writeFieldName("query");
        generator.writeStartObject();
        generator.writeFieldName("term");
        generator.writeStartObject();
        // note the difference here, teh field is analyzed2 and not analyzed as in the first doc, as well
        // as having a different value, though don't think it matters
        generator.writeStringField("doc.payload.test_record_main.string_not_analyzed2__s", "bar");
        generator.writeEndObject();
        generator.writeEndObject();
        generator.writeEndObject();
        generator.close();

        parser = factory.createJsonParser(bos2.toByteArray());
        token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        assertEquals("query", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        assertEquals("term", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.START_OBJECT, token);
        token = parser.nextToken();
        assertToken(JsonToken.FIELD_NAME, token);
        // here we fail..., seems to be a problem with field caching factory level???
        // since we get the field name of the previous (bos1) document field value (withou the 2)
        assertEquals("doc.payload.test_record_main.string_not_analyzed2__s", parser.getCurrentName());
        token = parser.nextToken();
        assertToken(JsonToken.VALUE_STRING, token);
        assertEquals("bar", parser.getText());

        parser.close();
    }
    
    /*
    /**********************************************************
    /* Helper methods
    /**********************************************************
     */
    
    private final String CHARS_40 = "0123456789012345678901234567890123456789";
    
    private byte[] writeStringValues(boolean enableSharing, int COUNT) throws IOException
    {
        SmileFactory f = new SmileFactory();
        f.configure(SmileGenerator.Feature.WRITE_HEADER, true);
        f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, enableSharing);
        ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
        JsonGenerator gen = f.createJsonGenerator(out);
        gen.writeStartArray();
        Random rnd = new Random(COUNT);
        for (int i = 0; i < COUNT; ++i) {
            gen.writeString(generateString(rnd.nextInt()));
        }
        gen.writeEndArray();
        gen.close();
        return out.toByteArray();
    }

    private void verifyStringValues(byte[] json, int COUNT) throws IOException
    {
        SmileFactory f = new SmileFactory();
        JsonParser jp = f.createJsonParser(json);
        assertToken(JsonToken.START_ARRAY, jp.nextToken());
        Random rnd = new Random(COUNT);
        for (int i = 0; i < COUNT; ++i) {
            String str = generateString(rnd.nextInt());
            assertToken(JsonToken.VALUE_STRING, jp.nextToken());
            assertEquals(str, jp.getText());
        }
        assertToken(JsonToken.END_ARRAY, jp.nextToken());
    }

    private String generateString(int rawNr)
    {
        int nr = rawNr % 1100;
        // Actually, let's try longer ones too
        String str = "some kind of String value we use"+nr;
        if (nr > 900) {
            str += CHARS_40;
        }
        return str;
    }
}