Rev 176 | Blame | Compare with Previous | Last modification | View Log | RSS feed
/** Copyright (c) 2009.** This program is free software; you can redistribute it and/or modify* it under the terms of the GNU General Public License version 3 as* published by the Free Software Foundation.** This program is distributed in the hope that it will be useful, but* WITHOUT ANY WARRANTY; without even the implied warranty of* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU* General Public License for more details.*/package uk.me.parabola.splitter;import crosby.binary.file.BlockInputStream;import org.xmlpull.v1.XmlPullParserException;import uk.me.parabola.splitter.args.ParamParser;import uk.me.parabola.splitter.args.SplitterParams;import uk.me.parabola.splitter.geo.City;import uk.me.parabola.splitter.geo.CityFinder;import uk.me.parabola.splitter.geo.CityLoader;import uk.me.parabola.splitter.geo.DefaultCityFinder;import uk.me.parabola.splitter.geo.DummyCityFinder;import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.PrintWriter;import java.io.Reader;import java.nio.charset.Charset;import java.util.Date;import java.util.List;import java.util.Map;import java.util.Set;/*** Splitter for OSM files with the purpose of providing input files for mkgmap.* <p/>* The input file is split so that no piece has more than a given number of nodes in it.** @author Steve Ratcliffe*/public class Main {private static final String DEFAULT_DIR = ".";// We can only process a maximum of 255 areas at a time because we// compress an area ID into 8 bits to save memory (and 0 is reserved)private int maxAreasPerPass;// A list of the OSM files to parse.private List<String> filenames;// The description to write into the template.args file.private String description;// The starting map ID.private int mapId;// The amount in map units that tiles overlap (note that the final img's will not overlap// but the input files do).private int overlapAmount;// The max number of nodes that will appear in a single file.private int maxNodes;// The maximum resolution of the map to be produced by mkgmap. This is a value in the range// 0-24. Higher numbers mean higher detail. The resolution determines how the tiles must// be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to// multiples of 2 ^ (24 - 13) = 2048 map units, and their widths and heights must be a multiple// of 2 * 2 ^ (24 - 13) = 4096 units. The tile widths and height multiples are double the tile// alignment because the center point of the tile is stored, and that must be aligned the// same as the tile edges are.private int resolution;// Whether or not to trim tiles of any empty space around their edges.private boolean trim;// This gets set if no osm file is supplied as a parameter and the cache is empty.private boolean useStdIn;// Set if there is a previous area file given on the command line.private AreaList areaList;// Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,// or they're all mixed up. Running with mixed enabled takes longer.private boolean mixed;// The path where the results are written out to.private File fileOutputDir;// A GeoNames file to use for naming the tiles.private String geoNamesFile;// How often (in seconds) to provide JVM status information. Zero = no information.private int statusFreq;// Whether to use the density map. Disabling this (not recommended) causes the splitter to// revert to using legacy mode which takes MUCH more memory during phase one.private boolean densityMap;private String kmlOutputFile;// The maximum number of threads the splitter should use.private int maxThreads;// The output typeprivate boolean pbfOutput;public static void main(String[] args) {Main m = new Main();m.start(args);}private void start(String[] args) {readArgs(args);if (statusFreq > 0) {JVMHealthMonitor.start(statusFreq);}long start = System.currentTimeMillis();System.out.println("Time started: " + new Date());try {split();} catch (IOException e) {System.err.println("Error opening or reading file " + e);e.printStackTrace();} catch (XmlPullParserException e) {System.err.println("Error parsing xml from file " + e);e.printStackTrace();}System.out.println("Time finished: " + new Date());System.out.println("Total time taken: " + (System.currentTimeMillis() - start) / 1000 + 's');}private void split() throws IOException, XmlPullParserException {File outputDir = fileOutputDir;if (!outputDir.exists()) {System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");if (!outputDir.mkdirs()) {System.err.println("Unable to create output directory! Using default directory instead");fileOutputDir = new File(DEFAULT_DIR);}} else if (!outputDir.isDirectory()) {System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");fileOutputDir = new File(DEFAULT_DIR);}if (filenames.isEmpty()) {if (areaList == null) {throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified");} else {int areaCount = areaList.getAreas().size();int passes = getAreasPerPass(areaCount);if (passes > 1) {throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes+ " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')');}useStdIn = true;}}if (areaList == null) {int alignment = 1 << (24 - resolution);System.out.println("Map is being split for resolution " + resolution + ':');System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units");System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment * 2) + " map units wide and high");areaList = calculateAreas();for (Area area : areaList.getAreas()) {area.setMapId(mapId++);}nameAreas();areaList.write(new File(fileOutputDir, "areas.list").getPath());} else {nameAreas();}List<Area> areas = areaList.getAreas();System.out.println(areas.size() + " areas:");for (Area area : areas) {System.out.print("Area " + area.getMapId() + " covers " + area.toHexString());if (area.getName() != null)System.out.print(' ' + area.getName());System.out.println();}if (kmlOutputFile != null) {File out = new File(kmlOutputFile);if (!out.isAbsolute())kmlOutputFile = new File(fileOutputDir, kmlOutputFile).getPath();System.out.println("Writing KML file to " + kmlOutputFile);areaList.writeKml(kmlOutputFile);}writeAreas(areas);writeArgsFile(areas);}private int getAreasPerPass(int areaCount) {return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass);}/*** Deal with the command line arguments.*/private void readArgs(String[] args) {ParamParser parser = new ParamParser();SplitterParams params = parser.parse(SplitterParams.class, args);if (!parser.getErrors().isEmpty()) {System.out.println();System.out.println("Invalid parameter(s):");for (String error : parser.getErrors()) {System.out.println(" " + error);}System.out.println();parser.displayUsage();System.exit(-1);}for (Map.Entry<String, Object> entry : parser.getConvertedParams().entrySet()) {String name = entry.getKey();Object value = entry.getValue();System.out.println(name + '=' + (value == null ? "" : value));}mapId = params.getMapid();overlapAmount = params.getOverlap();maxNodes = params.getMaxNodes();description = params.getDescription();geoNamesFile = params.getGeonamesFile();resolution = params.getResolution();trim = !params.isNoTrim();String output = params.getOutput();// Remove warning and make the default pbf after a while.if (output.equals("unset")) {System.err.println("\n\n**** WARNING: the default output type has changed to pbf, use --output=xml for .osm.gz files\n");output = "pbf";}if(!output.equals("xml") && !output.equals("pbf")) {System.err.println("The --output parameter must be either xml or pbf. Resetting to xml.");}pbfOutput = "pbf".equals(output);if (resolution < 1 || resolution > 24) {System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");resolution = 13;}mixed = params.isMixed();statusFreq = params.getStatusFreq();String outputDir = params.getOutputDir();fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);maxAreasPerPass = params.getMaxAreas();if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");maxAreasPerPass = 255;}kmlOutputFile = params.getWriteKml();densityMap = !params.isLegacyMode();if (!densityMap) {System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");}maxThreads = params.getMaxThreads().getCount();filenames = parser.getAdditionalParams();String splitFile = params.getSplitFile();if (splitFile != null) {try {areaList = new AreaList();areaList.read(splitFile);areaList.dump();} catch (IOException e) {areaList = null;System.err.println("Could not read area list file");e.printStackTrace();}}}/*** Calculate the areas that we are going to split into by getting the total area and* then subdividing down until each area has at most max-nodes nodes in it.*/private AreaList calculateAreas() throws IOException, XmlPullParserException {MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();MapProcessor processor = nodes;processMap(processor);//MapReader mapReader = processMap(processor);//System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +// Utils.format(mapReader.getWayCount()) + " ways and " +// Utils.format(mapReader.getRelationCount()) + " relations were processed ");System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));//System.out.println("Min node ID = " + mapReader.getMinNodeId());//System.out.println("Max node ID = " + mapReader.getMaxNodeId());System.out.println("Time: " + new Date());Area exactArea = nodes.getExactArea();SplittableArea splittableArea = nodes.getRoundedArea(resolution);System.out.println("Exact map coverage is " + exactArea);System.out.println("Trimmed and rounded map coverage is " + splittableArea.getBounds());System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");List<Area> areas = splittableArea.split(maxNodes);return new AreaList(areas);}private void nameAreas() throws IOException {CityFinder cityFinder;if (geoNamesFile != null) {CityLoader cityLoader = new CityLoader(true);List<City> cities = cityLoader.load(geoNamesFile);cityFinder = new DefaultCityFinder(cities);} else {cityFinder = new DummyCityFinder();}for (Area area : areaList.getAreas()) {// Decide what to call the areaSet<City> found = cityFinder.findCities(area);City bestMatch = null;for (City city : found) {if (bestMatch == null || city.getPopulation() > bestMatch.getPopulation()) {bestMatch = city;}}if (bestMatch != null)area.setName(bestMatch.getCountryCode() + '-' + bestMatch.getName());elsearea.setName(description);}}/*** Second pass, we have the areas so parse the file(s) again and write out each element* to the file(s) that should contain it.** @param areaList Area list determined on the first pass.*/private void writeAreas(List<Area> areas) throws IOException, XmlPullParserException {System.out.println("Writing out split osm files " + new Date());int numPasses = getAreasPerPass(areas.size());int areasPerPass = (int) Math.ceil((double) areas.size() / (double) numPasses);if (numPasses > 1) {System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");} else {System.out.println("Processing " + areas.size() + " areas in a single pass");}for (int i = 0; i < numPasses; i++) {OSMWriter[] currentWriters = new OSMWriter[Math.min(areasPerPass, areas.size() - i * areasPerPass)];for (int j = 0; j < currentWriters.length; j++) {Area area = areas.get(i * areasPerPass + j);currentWriters[j] = pbfOutput ? new BinaryMapWriter(area, fileOutputDir) : new OSMXMLWriter(area, fileOutputDir);currentWriters[j].initForWrite(area.getMapId(), overlapAmount);}System.out.println("Starting pass " + (i + 1) + " of " + numPasses + ", processing " + currentWriters.length +" areas (" + areas.get(i * areasPerPass).getMapId() + " to " +areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);processMap(processor);//System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +// Utils.format(mapReader.getWayCount()) + " ways, " +// Utils.format(mapReader.getRelationCount()) + " relations");}}private void processMap(MapProcessor processor) throws XmlPullParserException {// Create both an XML reader and a binary reader, Dispatch each input to the// Appropriate parser.OSMParser parser = new OSMParser(processor, mixed);if (useStdIn) {System.out.println("Reading osm data from stdin...");Reader reader = new InputStreamReader(System.in, Charset.forName("UTF-8"));parser.setReader(reader);try {try {parser.parse();} finally {reader.close();}} catch (IOException e) {e.printStackTrace();}}for (String filename : filenames) {System.out.println("Processing " + filename);try {if (filename.endsWith(".osm.pbf")) {// Is it a binary file?File file = new File(filename);BlockInputStream blockinput = (new BlockInputStream(new FileInputStream(file), new BinaryMapParser(processor)));try {blockinput.process();} finally {blockinput.close();}} else {// No, try XML.Reader reader = Utils.openFile(filename, maxThreads > 1);parser.setReader(reader);try {parser.parse();} finally {reader.close();}}} catch (FileNotFoundException e) {e.printStackTrace();} catch (XmlPullParserException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}}processor.endMap();}/*** Write a file that can be given to mkgmap that contains the correct arguments* for the split file pieces. You are encouraged to edit the file and so it* contains a template of all the arguments that you might want to use.*/protected void writeArgsFile(List<Area> areas) {PrintWriter w;try {w = new PrintWriter(new FileWriter(new File(fileOutputDir, "template.args")));} catch (IOException e) {System.err.println("Could not write template.args file");return;}w.println("#");w.println("# This file can be given to mkgmap using the -c option");w.println("# Please edit it first to add a description of each map.");w.println("#");w.println();w.println("# You can set the family id for the map");w.println("# family-id: 980");w.println("# product-id: 1");w.println();w.println("# Following is a list of map tiles. Add a suitable description");w.println("# for each one.");for (Area a : areas) {w.println();w.format("mapname: %08d\n", a.getMapId());if (a.getName() == null)w.println("# description: OSM Map");elsew.println("description: " + a.getName());if(pbfOutput)w.format("input-file: %08d.osm.pbf\n", a.getMapId());elsew.format("input-file: %08d.osm.gz\n", a.getMapId());}w.println();w.close();}}