Index: .idea/runConfigurations/mkgmap.xml
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- .idea/runConfigurations/mkgmap.xml	(revision 2687)
+++ .idea/runConfigurations/mkgmap.xml	(revision )
@@ -3,7 +3,7 @@
     <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
     <option name="MAIN_CLASS_NAME" value="uk.me.parabola.mkgmap.main.Main" />
     <option name="VM_PARAMETERS" value="-ea -Xmx1000m" />
-    <option name="PROGRAM_PARAMETERS" value="--tdbfile --index --code-page=65001 --route u.osm" />
+    <option name="PROGRAM_PARAMETERS" value="--tdbfile --index --route /opt/data/uk-test-1.osm.gz" />
     <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
     <option name="ALTERNATIVE_JRE_PATH" value="" />
\ No newline at end of file
Index: src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java	(revision )
@@ -16,7 +16,13 @@
  */
 package uk.me.parabola.imgfmt.app.labelenc;
 
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.util.Arrays;
 import java.util.Locale;
 
 /**
@@ -27,20 +33,24 @@
  */
 public class AnyCharsetEncoder extends BaseEncoder implements CharacterEncoder {
 
-	private final Charset charSet;
+	private final CharsetEncoder encoder;
+	private final Transliterator transliterator;
 
-	public AnyCharsetEncoder(String cs) {
+	public AnyCharsetEncoder(String cs, Transliterator transliterator) {
+		this.transliterator = transliterator;
 		prepareForCharacterSet(cs);
-		if (isCharsetSupported())
-			charSet = Charset.forName(cs);
-		else
-			charSet = null;
+		if (isCharsetSupported()) {
+			encoder = Charset.forName(cs).newEncoder();
+			encoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+		} else {
+			encoder = null;
-	}
+		}
+	}
 
 	public EncodedText encodeText(String text) {
 		if (text == null)
 			return NO_TEXT;
-		
+
 		if (!isCharsetSupported())
 			return simpleEncode(text);
 
@@ -50,10 +60,33 @@
 		else
 			ucText = text;
 
-		byte[] bytes = ucText.getBytes(charSet);
-		byte[] res = new byte[bytes.length + 1];
-		System.arraycopy(bytes, 0, res, 0, bytes.length);
+		byte[] bytes = new byte[(int) (ucText.length() * encoder.maxBytesPerChar()) + 10];
+		ByteBuffer bb = ByteBuffer.wrap(bytes);
+		CharBuffer charBuffer = CharBuffer.wrap(ucText);
 
+		CoderResult result;
+
+		do {
+			result = encoder.encode(charBuffer, bb, true);
+			if (result.isUnmappable()) {
+				char c = charBuffer.get();
+				String s = String.valueOf(c);
+
+				s = transliterator.transliterate(s);
+
+				for (int i = 0; i < s.length(); i++)
+					bb.put((byte) s.charAt(i));
+			}
+		} while (result != CoderResult.UNDERFLOW);
+
+		// We need it to be null terminated but also to trim any extra memory from the allocated
+		// buffer.
+		byte[] res = Arrays.copyOf(bytes, bb.position() + 1);
 		return new EncodedText(res, res.length);
+	}
+
+	public void setUpperCase(boolean upperCase) {
+		super.setUpperCase(upperCase);
+		transliterator.forceUppercase(upperCase);
 	}
 }
Index: src/uk/me/parabola/imgfmt/app/srt/Sort.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/srt/Sort.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/srt/Sort.java	(revision )
@@ -337,7 +337,7 @@
 		sort.charset = Charset.forName("ascii");
 		sort.encoder = sort.charset.newEncoder();
 		sort.setDescription("Default sort");
-		sort.setCodepage(codepage == 0? 1252: codepage);
+		sort.setCodepage(codepage);
 		return sort;
 	}
 
Index: src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java	(revision )
@@ -31,7 +31,6 @@
 	private int encodingType;
 	private CharacterEncoder encoder;
 	private CharacterDecoder decoder;
-	private Transliterator transliterator = new TableTransliterator("ascii");
 
 	protected void setEncoder(CharacterEncoder encoder) {
 		this.encoder = encoder;
@@ -65,14 +64,6 @@
 		this.codepage = codepage;
 	}
 
-	public Transliterator getTransliterator() {
-		return transliterator;
-	}
-
-	public void setTransliterator(Transliterator transliterator) {
-		this.transliterator = transliterator;
-	}
-
 	/**
 	 * Create a CharacterEncoder for the given charset option.  Note that this
 	 * routine also writes to the lblHeader parameter to set the encoding type.
@@ -81,129 +72,90 @@
 	 */
 	public static CodeFunctions createEncoderForLBL(String charset) {
 		CodeFunctions funcs = new CodeFunctions();
-
 		if ("ascii".equals(charset)) {
 			funcs.setEncodingType(ENCODING_FORMAT6);
 			funcs.setEncoder(new Format6Encoder());
-			funcs.setTransliterator(getDefaultTransliterator());
 			funcs.setDecoder(new Format6Decoder());
-		} else if ("latin1".equals(charset)) {
+		} else if ("cp1252".equals(charset) || "latin1".equals(charset)) {
 			funcs.setEncodingType(ENCODING_FORMAT9);
-			funcs.setEncoder(new LatinEncoder());
-			funcs.setTransliterator(new TableTransliterator("latin1"));
+			funcs.setEncoder(new AnyCharsetEncoder("cp1252", new TableTransliterator("latin1")));
 			funcs.setDecoder(new AnyCharsetDecoder("cp1252"));
 			funcs.setCodepage(1252);
-		} else if ("unicode".equals(charset)) {
+		} else if ("cp65001".equals(charset) || "unicode".equals(charset)) {
 			funcs.setEncodingType(ENCODING_FORMAT10);
 			funcs.setEncoder(new Utf8Encoder());
 			funcs.setDecoder(new Utf8Decoder());
-		} else if ("cp65001".equals(charset)) {
-			funcs.setEncodingType(ENCODING_FORMAT10);
-			funcs.setEncoder(new Utf8Encoder());
-			funcs.setDecoder(new Utf8Decoder());
-			funcs.setTransliterator(new NullTransliterator());
 			funcs.setCodepage(65001);
 		} else if ("simple8".equals(charset)) {
 			funcs.setEncodingType(ENCODING_FORMAT9);
 			funcs.setEncoder(new Simple8Encoder());
 		} else if ("cp932".equals(charset) || "ms932".equals(charset)) {
 			funcs.setEncodingType(ENCODING_FORMAT10);
-			funcs.setEncoder(new AnyCharsetEncoder("ms932"));
+			funcs.setEncoder(new AnyCharsetEncoder("ms932", new SparseTransliterator("nomacron")));
 			funcs.setDecoder(new AnyCharsetDecoder("ms932"));
-			funcs.setTransliterator(new SparseTransliterator("nomacron"));
 			funcs.setCodepage(932);
 		} else {
 			funcs.setEncodingType(ENCODING_FORMAT9);
 			funcs.setDecoder(new AnyCharsetDecoder(charset));
-			funcs.setEncoder(new AnyCharsetEncoder(charset));
-			guessCodepage(funcs, charset);
+			funcs.setEncoder(new AnyCharsetEncoder(charset, new TableTransliterator("ascii")));
+			funcs.setCodepage(guessCodepage(charset));
 		}
 
 		return funcs;
 	}
 
 	/**
+	 * Sets encoding functions for a given format and code page.  This is used
+	 * when reading from an existing file.
+	 *
+	 * @param format The format from the lbl header.
+	 * @param codePage The codepage found in the header.
+	 * @return The various character set parameters that will be needed.
+	 */
+	public static CodeFunctions createEncoderForLBL(int format, int codePage) {
+		CodeFunctions funcs;
+
+		if (format == ENCODING_FORMAT6) {
+			funcs = createEncoderForLBL("ascii");
+		} else {
+			funcs = createEncoderForLBL("cp" + codePage);
+		}
+
+		return funcs;
+	}
+
+	/**
 	 * Guess the code page from the given charset.  Only works with things
 	 * like cp1252, windows-1252 and some well known ones.
-	 * @param funcs The code page functions.
 	 * @param charset The charset that was given.
 	 */
-	private static void guessCodepage(CodeFunctions funcs, String charset) {
+	private static int guessCodepage(String charset) {
 		String cs = charset.toLowerCase();
-		Transliterator transliterator = new NullTransliterator();
 		if (cs.startsWith("cp")) {
 			try {
-				funcs.setCodepage(Integer.parseInt(charset.substring(2)));
-				if (cs.equals("cp1252"))
-					transliterator = new TableTransliterator("latin1");
+				return Integer.parseInt(charset.substring(2));
 			} catch (NumberFormatException e) {
 				// wasn't in the right form
 				throw new ExitException("Invalid character set: " + cs);
 			}
 		} else if (cs.startsWith("windows-")) {
 			try {
-				funcs.setCodepage(Integer.parseInt(charset.substring(8)));
+				return Integer.parseInt(charset.substring(8));
 			} catch (NumberFormatException e) {
 				// wasn't in the right form to guess
+				throw new ExitException("Invalid character set: " + cs);
 			}
 		} else if (cs.equals("latin1")) {
-			funcs.setCodepage(1252);
-			transliterator = new TableTransliterator("latin1");
+			return 1252;
 		}
-		funcs.setTransliterator(transliterator);
+		return 0;
 	}
 
-	/**
-	 * Sets encoding functions for a given format and code page.  This is used
-	 * when reading from an existing file.
-	 *
-	 *
-	 * @param format The format from the lbl header.
-	 * @param codePage The codepage found in the header.
-	 * @return The various character set parameters that will be needed.
-	 */
-	public static CodeFunctions createEncoderForLBL(int format, int codePage) {
-		CodeFunctions funcs = new CodeFunctions();
-
-		if (format == ENCODING_FORMAT6) {
-			funcs.setEncodingType(ENCODING_FORMAT6);
-			funcs.setEncoder(new Format6Encoder());
-			funcs.setDecoder(new Format6Decoder());
-		} else if (codePage == 932) {
-			funcs.setEncodingType(ENCODING_FORMAT10);
-			funcs.setEncoder(new AnyCharsetEncoder("ms932"));
-			funcs.setDecoder(new AnyCharsetDecoder("ms932"));
-		} else if (format == ENCODING_FORMAT9) {
-			funcs.setEncodingType(ENCODING_FORMAT9);
-			String cpName = "cp" + codePage;
-			if (codePage == 1252)
-				funcs.setEncoder(new LatinEncoder());
-			else
-				funcs.setEncoder(new AnyCharsetEncoder(cpName));
-			guessCodepage(funcs, cpName);
-
-			funcs.setDecoder(new AnyCharsetDecoder(cpName));
-		} else {
-			// TODO TEMP...
-			funcs.setEncodingType(ENCODING_FORMAT9);
-			funcs.setEncoder(new AnyCharsetEncoder("cp1252"));
-			funcs.setDecoder(new AnyCharsetDecoder("cp1252"));
-		}
-
-		return funcs;
-	}
-
 	public static CharacterEncoder getDefaultEncoder() {
 		return new Format6Encoder();
 	}
 
 	public static CharacterDecoder getDefaultDecoder() {
 		return new Format6Decoder();
-	}
-
-	public static Transliterator getDefaultTransliterator() {
-		TableTransliterator ascii = new TableTransliterator("ascii");
-		ascii.forceUppercase(true);
-		return ascii;
 	}
 }
Index: test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java	(revision 2687)
+++ test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java	(revision )
@@ -58,7 +58,7 @@
 		CodeFunctions functions = CodeFunctions.createEncoderForLBL(6, 0);
 
 		CharacterEncoder encoder = functions.getEncoder();
-		Transliterator transliterator = functions.getTransliterator();
+		Transliterator transliterator = new TableTransliterator("ascii");
 		EncodedText text = encoder.encodeText(transliterator.transliterate("K��rnerstra��e, Velkomeze������sk��, Sk��lav��r��ustigur"));
 
 		CharacterDecoder decoder = functions.getDecoder();
@@ -72,15 +72,15 @@
 	}
 
 	/**
-	 * Transliteration when going to ascii in format 6.  This was originally
-	 * the only place where transliteration was available.
-	 */
+	* Transliteration when going to ascii in format 6.  This was originally
+	* the only place where transliteration was available.
+	*/
 	@Test
 	public void testTransliterateLatin() {
 		CodeFunctions functions = CodeFunctions.createEncoderForLBL("latin1");
 
 		CharacterEncoder encoder = functions.getEncoder();
-		Transliterator transliterator = functions.getTransliterator();
+		Transliterator transliterator = new TableTransliterator("latin1");
 		EncodedText text = encoder.encodeText(transliterator.transliterate("K��rnerstra��e, Velkomeze������sk��, Sk��lav��r��ustigur"));
 
 		CharacterDecoder decoder = functions.getDecoder();
@@ -88,7 +88,7 @@
 		for (int i = 0; i < text.getLength(); i++) {
 			decoder.addByte(ctext[i]);
 		}
-		
+
 		String result = decoder.getText().getText();
 		assertEquals("transliterated text", "K��rnerstra��e, Velkomezer��csk��, Sk��lav��r��ustigur", result);
 	}
Index: src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java	(revision 2687)
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2009.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
-package uk.me.parabola.imgfmt.app.labelenc;
-
-import java.nio.charset.Charset;
-import java.util.Locale;
-
-/**
- * An encoder for latin script
- * @author Steve Ratcliffe
- */
-public class LatinEncoder extends BaseEncoder implements CharacterEncoder {
-	private final Charset latinCharset = Charset.forName("latin1");
-
-	public EncodedText encodeText(String t) {
-		String text;
-		if (t != null && isUpperCase())
-			text = t.toUpperCase(Locale.ENGLISH);
-		else
-			text = t;
-		// Need to add a null character at the end of the string for this format.
-		String zText = text + "\000";
-
-		byte[] chars = zText.getBytes(latinCharset);
-		return new EncodedText(chars, chars.length);
-	}
-}
Index: src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java	(revision )
@@ -48,6 +48,8 @@
 		"xxxxxxxxxx:;<=>?" +	// 0x10-0x1F
 		"xxxxxxxxxxx[\\]^_";	// 0x20-0x2F
 
+	private final Transliterator transliterator = new TableTransliterator("ascii");
+
 	/**
 	 * Encode the text into the 6 bit format.  See the class level notes.
 	 *
@@ -56,10 +58,10 @@
 	 * some escape sequences will be present.
 	 */
 	public EncodedText encodeText(String text) {
-		if (text == null || text.length() == 0)
+		if (text == null || text.isEmpty())
 			return NO_TEXT;
 
-		String s = text.toUpperCase(Locale.ENGLISH);
+		String s = transliterator.transliterate(text).toUpperCase(Locale.ENGLISH);
 
 		// Allocate more than enough space on average for the label.
 		// if you overdo it then it will waste a lot of space , but
Index: src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java	(revision 2687)
+++ src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java	(revision )
@@ -28,7 +28,6 @@
 import uk.me.parabola.imgfmt.app.labelenc.BaseEncoder;
 import uk.me.parabola.imgfmt.app.labelenc.CharacterEncoder;
 import uk.me.parabola.imgfmt.app.labelenc.CodeFunctions;
-import uk.me.parabola.imgfmt.app.labelenc.Transliterator;
 import uk.me.parabola.imgfmt.app.srt.Sort;
 import uk.me.parabola.imgfmt.app.trergn.Subdivision;
 import uk.me.parabola.imgfmt.fs.ImgChannel;
@@ -48,7 +47,6 @@
 	private static final Logger log = Logger.getLogger(LBLFile.class);
 
 	private CharacterEncoder textEncoder = CodeFunctions.getDefaultEncoder();
-	private Transliterator transliterator = CodeFunctions.getDefaultTransliterator();
 
 	private final Map<String, Label> labelCache = new HashMap<String, Label>();
 
@@ -102,13 +100,10 @@
 		
 		lblHeader.setEncodingType(cfuncs.getEncodingType());
 		textEncoder = cfuncs.getEncoder();
-		transliterator = cfuncs.getTransliterator();
 		if (forceUpper && textEncoder instanceof BaseEncoder) {
 			BaseEncoder baseEncoder = (BaseEncoder) textEncoder;
 			baseEncoder.setUpperCase(true);
 		}
-		if (forceUpper)
-			transliterator.forceUppercase(true);
 	}
 
 	public void setEncoder(int encodingType, int codepage ) {
@@ -116,18 +111,16 @@
 		
 		lblHeader.setEncodingType(cfuncs.getEncodingType());
 		textEncoder = cfuncs.getEncoder();
-		transliterator = cfuncs.getTransliterator();
 	}
 	
 	/**
 	 * Add a new label with the given text.  Labels are shared, so that identical
 	 * text is always represented by the same label.
 	 *
-	 * @param inText The text of the label, it will be in uppercase.
+	 * @param text The text of the label, it will be in uppercase.
 	 * @return A reference to the created label.
 	 */
-	public Label newLabel(String inText) {
-		String text = transliterator.transliterate(inText);
+	public Label newLabel(String text) {
 		Label l = labelCache.get(text);
 		if (l == null) {
 			l = new Label(text);
