[mkgmap-dev] splitter patch for enabling uncompressed tile outputs

Mon Mar 22 08:56:06 GMT 2010

Hi Christoph,

I'll apply this patch in a week or so once I get back home. In the meantime 
though, I'd be interested to hear about the performance problems you're seeing 
with the splitter. Do you think it is due to the cpu time required for the 
compression, or something else? Why do you want to disable the gzipped output?

Regards,
Chris

> This is an OpenPGP/MIME signed message (RFC 2440 and 3156)
> --------------enig89B73E3FE17316F7A6ECBF28
> Content-Type: multipart/mixed;
> boundary="------------000707010409030508090004"
> This is a multi-part message in MIME format.
> --------------000707010409030508090004
> Content-Type: text/plain; charset=ISO-8859-15
> Content-Transfer-Encoding: quoted-printable
> Hello list,
> 
> I wanted to test some performance issues with splitter and so I needed
> to=
> switch of the gzip compression of the output tiles.
> Based on a patch that Thilo Hannemann wrote in may last year I made a
> new=
> patch for the latest splitter version now.
> Please can somebody submit this patch to the splitter svn? Maybe other
> pe= ople need this, too.
> 
> You can give a commandline switch like
> 
> splitter.jar --uncompressed
> 
> or
> 
> splitter.jar --uncompressed=3Dtrue
> 
> to switch of the compression for the output tiles. Default is to
> compress= =2E
> 
> Thanks!
> Christoph
> --------------000707010409030508090004
> Content-Type: text/x-patch;
> name="uncompressed.patch"
> Content-Transfer-Encoding: quoted-printable
> Content-Disposition: inline;
> filename="uncompressed.patch"
> Index: src/uk/me/parabola/splitter/args/SplitterParams.java
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
> --- src/uk/me/parabola/splitter/args/SplitterParams.java	(Revision
> 105)
> +++ src/uk/me/parabola/splitter/args/SplitterParams.java
> (Arbeitskopie)
> @@ -63,4 +63,7 @@
> =20
> @Option(description =3D "Don't trim empty space off the edges of
> tiles.=
> ")
> boolean isNoTrim();
> +
> +	@Option(description =3D "Don't gzip output tiles if --uncompressed
> is g=
> iven.")
> +	boolean isUncompressed();
> }
> Index: src/uk/me/parabola/splitter/Main.java
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
> --- src/uk/me/parabola/splitter/Main.java	(Revision 105)
> +++ src/uk/me/parabola/splitter/Main.java	(Arbeitskopie)
> @@ -77,6 +77,8 @@
> // Set if there is a previous area file given on the command line.
> private AreaList areaList;
> private boolean mixed;
> +	// The option "--uncompressed" will turn off compression of the
> output =
> data.
> +	private boolean uncompressed;
> // The path to the disk cache. If this is null, no cache will be
> genera=
> ted or used.
> private String diskCachePath;
> // Whether or not a new cache needs to be generated.
> @@ -215,6 +217,7 @@
> geoNamesFile =3D params.getGeonamesFile();
> resolution =3D params.getResolution();
> trim =3D !params.isNoTrim();
> +		uncompressed =3D params.isUncompressed();
> if (resolution < 1 || resolution > 24) {
> System.err.println("The --resolution parameter must be a value
> betwee=
> n 1 and 24. Resetting to 13.");
> resolution =3D 13;
> @@ -341,7 +344,7 @@
> for (int j =3D 0; j < currentWriters.length; j++) {
> Area area =3D areas.get(i * maxAreasPerPass + j);
> currentWriters[j] =3D new OSMWriter(area);
> -				currentWriters[j].initForWrite(area.getMapId(), overlapAmount);
> +				currentWriters[j].initForWrite(area.getMapId(), overlapAmount,
> uncom=
> pressed);
> }
> =20
> System.out.println("Starting pass " + (i + 1) + " of " +
> passesRequir=
> ed + ", processing " + currentWriters.length +
> @@ -431,7 +434,11 @@
> w.println("# description: OSM Map");
> else
> w.println("description: " + a.getName());
> -			w.format("input-file: %d.osm.gz\n", a.getMapId());
> +			if (uncompressed) {
> +				w.format("input-file: %d.osm\n", a.getMapId());
> +			} else {
> +				w.format("input-file: %d.osm.gz\n", a.getMapId());
> +			}
> }
> =20
> w.println();
> Index: src/uk/me/parabola/splitter/OSMWriter.java
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
> 3D=3D=
> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
> --- src/uk/me/parabola/splitter/OSMWriter.java	(Revision 105)
> +++ src/uk/me/parabola/splitter/OSMWriter.java	(Arbeitskopie)
> @@ -34,16 +34,26 @@
> this.bounds =3D bounds;
> }
> =20
> -	public void initForWrite(int mapId, int extra) {
> +	public void initForWrite(int mapId, int extra, boolean uncompressed)
> {
> extendedBounds =3D new Area(bounds.getMinLat() - extra,
> bounds.getMinLong() - extra,
> bounds.getMaxLat() + extra,
> bounds.getMaxLong() + extra);
> =20
> -		String filename =3D new Formatter().format(Locale.ROOT,
> "%08d.osm.gz",=
> mapId).toString();
> +		String filename;
> +		if (uncompressed) {
> +			filename =3D new Formatter().format(Locale.ROOT, "%08d.osm",
> mapId).t=
> oString();
> +		} else {
> +			filename =3D new Formatter().format(Locale.ROOT, "%08d.osm.gz",
> mapId=
> ).toString();
> +		}
> try {
> FileOutputStream fos =3D new FileOutputStream(filename);
> -			OutputStream zos =3D new GZIPOutputStream(fos);
> +			OutputStream zos;
> +			if (uncompressed) {
> +				zos =3D fos;
> +			} else {
> +				zos =3D new GZIPOutputStream(fos);
> +			}
> writer =3D new OutputStreamWriter(zos, "utf-8");
> writeHeader();
> } catch (IOException e) {
> --------------000707010409030508090004--
> 
> --------------enig89B73E3FE17316F7A6ECBF28
> Content-Type: application/pgp-signature; name="signature.asc"
> Content-Description: OpenPGP digital signature
> Content-Disposition: attachment; filename="signature.asc"
> -----BEGIN PGP SIGNATURE-----
>> Version: GnuPG v1.4.9 (GNU/Linux)
>> 
>> iEYEARECAAYFAkuk6IcACgkQkASFvss5Ug8RggCdFhAlIB/utda48A3dBUdIWC6U
>> 5g0An18MCNjkncQs19hv+YQU45OjyJQi
>> =TPP8
> -----END PGP SIGNATURE-----
>> --------------enig89B73E3FE17316F7A6ECBF28--
>> 