Subversion Repositories splitter

Rev

Rev 176 | Blame | Compare with Previous | Last modification | View Log | RSS feed

/*
 * Copyright (c) 2009.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 */


package uk.me.parabola.splitter;

import crosby.binary.file.BlockInputStream;
import org.xmlpull.v1.XmlPullParserException;
import uk.me.parabola.splitter.args.ParamParser;
import uk.me.parabola.splitter.args.SplitterParams;
import uk.me.parabola.splitter.geo.City;
import uk.me.parabola.splitter.geo.CityFinder;
import uk.me.parabola.splitter.geo.CityLoader;
import uk.me.parabola.splitter.geo.DefaultCityFinder;
import uk.me.parabola.splitter.geo.DummyCityFinder;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Splitter for OSM files with the purpose of providing input files for mkgmap.
 * <p/>
 * The input file is split so that no piece has more than a given number of nodes in it.
 *
 * @author Steve Ratcliffe
 */

public class Main {
        private static final String DEFAULT_DIR = ".";

        // We can only process a maximum of 255 areas at a time because we
        // compress an area ID into 8 bits to save memory (and 0 is reserved)
        private int maxAreasPerPass;

        // A list of the OSM files to parse.
        private List<String> filenames;

        // The description to write into the template.args file.
        private String description;

        // The starting map ID.
        private int mapId;

        // The amount in map units that tiles overlap (note that the final img's will not overlap
        // but the input files do).
        private int overlapAmount;

        // The max number of nodes that will appear in a single file.
        private int maxNodes;

        // The maximum resolution of the map to be produced by mkgmap. This is a value in the range
        // 0-24. Higher numbers mean higher detail. The resolution determines how the tiles must
        // be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to
        // multiples of 2 ^ (24 - 13) = 2048 map units, and their widths and heights must be a multiple
        // of 2 * 2 ^ (24 - 13) = 4096 units. The tile widths and height multiples are double the tile
        // alignment because the center point of the tile is stored, and that must be aligned the
        // same as the tile edges are.
        private int resolution;

        // Whether or not to trim tiles of any empty space around their edges.
        private boolean trim;
        // This gets set if no osm file is supplied as a parameter and the cache is empty.
        private boolean useStdIn;
        // Set if there is a previous area file given on the command line.
        private AreaList areaList;
        // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
        // or they're all mixed up. Running with mixed enabled takes longer.
        private boolean mixed;
        // The path where the results are written out to.
        private File fileOutputDir;
        // A GeoNames file to use for naming the tiles.
        private String geoNamesFile;
        // How often (in seconds) to provide JVM status information. Zero = no information.
        private int statusFreq;
        // Whether to use the density map. Disabling this (not recommended) causes the splitter to
        // revert to using legacy mode which takes MUCH more memory during phase one.
        private boolean densityMap;

        private String kmlOutputFile;
        // The maximum number of threads the splitter should use.
        private int maxThreads;
        // The output type
        private boolean pbfOutput;

        public static void main(String[] args) {

                Main m = new Main();
                m.start(args);
        }

        private void start(String[] args) {
                readArgs(args);
                if (statusFreq > 0) {
                        JVMHealthMonitor.start(statusFreq);
                }
                long start = System.currentTimeMillis();
                System.out.println("Time started: " + new Date());
                try {
                        split();
                } catch (IOException e) {
                        System.err.println("Error opening or reading file " + e);
                        e.printStackTrace();
                } catch (XmlPullParserException e) {
                        System.err.println("Error parsing xml from file " + e);
                        e.printStackTrace();
                }
                System.out.println("Time finished: " + new Date());
                System.out.println("Total time taken: " + (System.currentTimeMillis() - start) / 1000 + 's');
        }

        private void split() throws IOException, XmlPullParserException {

                File outputDir = fileOutputDir;
                if (!outputDir.exists()) {
                        System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
                        if (!outputDir.mkdirs()) {
                                System.err.println("Unable to create output directory! Using default directory instead");
                                fileOutputDir = new File(DEFAULT_DIR);
                        }
                } else if (!outputDir.isDirectory()) {
                        System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
                        fileOutputDir = new File(DEFAULT_DIR);
                }

                if (filenames.isEmpty()) {
                        if (areaList == null) {
                                throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified");
                        } else {
                                int areaCount = areaList.getAreas().size();
                                int passes = getAreasPerPass(areaCount);
                                if (passes > 1) {
                                        throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes
                                                        + " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')');
                                }
                                useStdIn = true;
                        }
                }

                if (areaList == null) {
                        int alignment = 1 << (24 - resolution);
                        System.out.println("Map is being split for resolution " + resolution + ':');
                        System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units");
                        System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment * 2) + " map units wide and high");
                        areaList = calculateAreas();
                        for (Area area : areaList.getAreas()) {
                                area.setMapId(mapId++);
                        }
                        nameAreas();
                        areaList.write(new File(fileOutputDir, "areas.list").getPath());
                } else {
                        nameAreas();
                }

                List<Area> areas = areaList.getAreas();
                System.out.println(areas.size() + " areas:");
                for (Area area : areas) {
                        System.out.print("Area " + area.getMapId() + " covers " + area.toHexString());
                        if (area.getName() != null)
                                System.out.print(' ' + area.getName());
                        System.out.println();
                }

                if (kmlOutputFile != null) {
                        File out = new File(kmlOutputFile);
                        if (!out.isAbsolute())
                                kmlOutputFile = new File(fileOutputDir, kmlOutputFile).getPath();
                        System.out.println("Writing KML file to " + kmlOutputFile);
                        areaList.writeKml(kmlOutputFile);
                }

                writeAreas(areas);
                writeArgsFile(areas);
        }

        private int getAreasPerPass(int areaCount) {
                return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass);
        }

        /**
         * Deal with the command line arguments.
         */

        private void readArgs(String[] args) {
                ParamParser parser = new ParamParser();
                SplitterParams params = parser.parse(SplitterParams.class, args);

                if (!parser.getErrors().isEmpty()) {
                        System.out.println();
                        System.out.println("Invalid parameter(s):");
                        for (String error : parser.getErrors()) {
                                System.out.println("  " + error);
                        }
                        System.out.println();
                        parser.displayUsage();
                        System.exit(-1);
                }

                for (Map.Entry<String, Object> entry : parser.getConvertedParams().entrySet()) {
                        String name = entry.getKey();
                        Object value = entry.getValue();
                        System.out.println(name + '=' + (value == null ? "" : value));
                }

                mapId = params.getMapid();
                overlapAmount = params.getOverlap();
                maxNodes = params.getMaxNodes();
                description = params.getDescription();
                geoNamesFile = params.getGeonamesFile();
                resolution = params.getResolution();
                trim = !params.isNoTrim();
                String output = params.getOutput();
                // Remove warning and make the default pbf after a while.
                if (output.equals("unset")) {
                        System.err.println("\n\n**** WARNING: the default output type has changed to pbf, use --output=xml for .osm.gz files\n");
                        output = "pbf";
                }
                if(!output.equals("xml") && !output.equals("pbf")) {
                        System.err.println("The --output parameter must be either xml or pbf. Resetting to xml.");
                }
                pbfOutput = "pbf".equals(output);
               
                if (resolution < 1 || resolution > 24) {
                        System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");
                        resolution = 13;
                }
                mixed = params.isMixed();
                statusFreq = params.getStatusFreq();
               
                String outputDir = params.getOutputDir();
                fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);

                maxAreasPerPass = params.getMaxAreas();
                if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {
                        System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");
                        maxAreasPerPass = 255;
                }
                kmlOutputFile = params.getWriteKml();
                densityMap = !params.isLegacyMode();
                if (!densityMap) {
                        System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");
                }

                maxThreads = params.getMaxThreads().getCount();
                filenames = parser.getAdditionalParams();
               
                String splitFile = params.getSplitFile();
                if (splitFile != null) {
                        try {
                                areaList = new AreaList();
                                areaList.read(splitFile);
                                areaList.dump();
                        } catch (IOException e) {
                                areaList = null;
                                System.err.println("Could not read area list file");
                                e.printStackTrace();
                        }
                }
        }

        /**
         * Calculate the areas that we are going to split into by getting the total area and
         * then subdividing down until each area has at most max-nodes nodes in it.
         */

        private AreaList calculateAreas() throws IOException, XmlPullParserException {

                MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();
                MapProcessor processor = nodes;

                processMap(processor);
                //MapReader mapReader = processMap(processor);

                //System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
                //                              Utils.format(mapReader.getWayCount()) + " ways and " +
                //                              Utils.format(mapReader.getRelationCount()) + " relations were processed ");

                System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));

                //System.out.println("Min node ID = " + mapReader.getMinNodeId());
                //System.out.println("Max node ID = " + mapReader.getMaxNodeId());

                System.out.println("Time: " + new Date());

                Area exactArea = nodes.getExactArea();
                SplittableArea splittableArea = nodes.getRoundedArea(resolution);
                System.out.println("Exact map coverage is " + exactArea);
                System.out.println("Trimmed and rounded map coverage is " + splittableArea.getBounds());
                System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");

                List<Area> areas = splittableArea.split(maxNodes);
                return new AreaList(areas);
        }

        private void nameAreas() throws IOException {
                CityFinder cityFinder;
                if (geoNamesFile != null) {
                        CityLoader cityLoader = new CityLoader(true);
                        List<City> cities = cityLoader.load(geoNamesFile);
                        cityFinder = new DefaultCityFinder(cities);
                } else {
                        cityFinder = new DummyCityFinder();
                }

                for (Area area : areaList.getAreas()) {
                        // Decide what to call the area
                        Set<City> found = cityFinder.findCities(area);
                        City bestMatch = null;
                        for (City city : found) {
                                if (bestMatch == null || city.getPopulation() > bestMatch.getPopulation()) {
                                        bestMatch = city;
                                }
                        }
                        if (bestMatch != null)
                                area.setName(bestMatch.getCountryCode() + '-' + bestMatch.getName());
                        else
                                area.setName(description);
                }
        }

        /**
         * Second pass, we have the areas so parse the file(s) again and write out each element
         * to the file(s) that should contain it.
         *
         * @param areaList Area list determined on the first pass.
         */

        private void writeAreas(List<Area> areas) throws IOException, XmlPullParserException {
                System.out.println("Writing out split osm files " + new Date());

                int numPasses = getAreasPerPass(areas.size());
                int areasPerPass = (int) Math.ceil((double) areas.size() / (double) numPasses);

                if (numPasses > 1) {
                        System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
                } else {
                        System.out.println("Processing " + areas.size() + " areas in a single pass");
                }

                for (int i = 0; i < numPasses; i++) {
                        OSMWriter[] currentWriters = new OSMWriter[Math.min(areasPerPass, areas.size() - i * areasPerPass)];
                        for (int j = 0; j < currentWriters.length; j++) {
                                Area area = areas.get(i * areasPerPass + j);
                                currentWriters[j] = pbfOutput ? new BinaryMapWriter(area, fileOutputDir) : new OSMXMLWriter(area, fileOutputDir);
                                currentWriters[j].initForWrite(area.getMapId(), overlapAmount);
                        }

                        System.out.println("Starting pass " + (i + 1) + " of " + numPasses + ", processing " + currentWriters.length +
                                                        " areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
                                                        areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');

                        MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
                        processMap(processor);
                        //System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
                        //                              Utils.format(mapReader.getWayCount()) + " ways, " +
                        //                              Utils.format(mapReader.getRelationCount()) + " relations");
                }
        }
       
        private void processMap(MapProcessor processor) throws XmlPullParserException {
                // Create both an XML reader and a binary reader, Dispatch each input to the
                // Appropriate parser.
                OSMParser parser = new OSMParser(processor, mixed);
                if (useStdIn) {
                        System.out.println("Reading osm data from stdin...");
                        Reader reader = new InputStreamReader(System.in, Charset.forName("UTF-8"));
                        parser.setReader(reader);
                        try {
                                try {
                                        parser.parse();
                                } finally {
                                        reader.close();
                                }
                        } catch (IOException e) {
                                e.printStackTrace();
                        }
                }

                for (String filename : filenames) {
                        System.out.println("Processing " + filename);
                        try {
                                if (filename.endsWith(".osm.pbf")) {
                                        // Is it a binary file?
                                        File file = new File(filename);
                                        BlockInputStream blockinput = (new BlockInputStream(
                                                        new FileInputStream(file), new BinaryMapParser(processor)));
                                        try {
                                                blockinput.process();
                                        } finally {
                                                blockinput.close();
                                        }
                                } else {
                                        // No, try XML.
                                        Reader reader = Utils.openFile(filename, maxThreads > 1);
                                        parser.setReader(reader);
                                        try {
                                                parser.parse();
                                        } finally {
                                                reader.close();
                                        }
                                }
                        } catch (FileNotFoundException e) {
                                e.printStackTrace();
                        } catch (XmlPullParserException e) {
                                e.printStackTrace();
                        } catch (IOException e) {
                                e.printStackTrace();
                        }
                }
                processor.endMap();
        }
       
        /**
         * Write a file that can be given to mkgmap that contains the correct arguments
         * for the split file pieces.  You are encouraged to edit the file and so it
         * contains a template of all the arguments that you might want to use.
         */

        protected void writeArgsFile(List<Area> areas) {
                PrintWriter w;
                try {
                        w = new PrintWriter(new FileWriter(new File(fileOutputDir, "template.args")));
                } catch (IOException e) {
                        System.err.println("Could not write template.args file");
                        return;
                }

                w.println("#");
                w.println("# This file can be given to mkgmap using the -c option");
                w.println("# Please edit it first to add a description of each map.");
                w.println("#");
                w.println();

                w.println("# You can set the family id for the map");
                w.println("# family-id: 980");
                w.println("# product-id: 1");

                w.println();
                w.println("# Following is a list of map tiles.  Add a suitable description");
                w.println("# for each one.");
                for (Area a : areas) {
                        w.println();
                        w.format("mapname: %08d\n", a.getMapId());
                        if (a.getName() == null)
                                w.println("# description: OSM Map");
                        else
                                w.println("description: " + a.getName());
                        if(pbfOutput)
                          w.format("input-file: %08d.osm.pbf\n", a.getMapId());
                        else
                          w.format("input-file: %08d.osm.gz\n", a.getMapId());
                }

                w.println();
                w.close();
        }
}