vcs.maemo.org Git - navit-package/blob - navit/osm2navit.c

   1 /**
   2  * Navit, a modular navigation system.
   3  * Copyright (C) 2005-2008 Navit Team
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the
  16  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  17  * Boston, MA  02110-1301, USA.
  18  */
  19
  20
  21 #define _FILE_OFFSET_BITS 64
  22 #define _LARGEFILE_SOURCE
  23 #define _LARGEFILE64_SOURCE
  24 #include <glib.h>
  25 #include <assert.h>
  26 #include <string.h>
  27 #include <signal.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <math.h>
  31 #include <getopt.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <sys/stat.h>
  35 #include <zlib.h>
  36 #include "item.h"
  37 #include "zipfile.h"
  38 #include "config.h"
  39 #ifdef HAVE_POSTGRESQL
  40 #include <libpq-fe.h>
  41 #endif
  42
  43 #define BUFFER_SIZE 1280
  44
  45 #define GENERATE_INDEX
  46
  47 #if 1
  48 #define debug_tile(x) 0
  49 #else
  50 #define debug_tile(x) (!strcmp(x,"bcdbd") || !strcmp(x,"bcdbd") || !strcmp(x,"bcdbda") || !strcmp(x,"bcdbdb") || !strcmp(x,"bcdbdba") || !strcmp(x,"bcdbdbb") || !strcmp(x,"bcdbdbba") || !strcmp(x,"bcdbdbaa") || !strcmp(x,"bcdbdbacaa") || !strcmp(x,"bcdbdbacab") || !strcmp(x,"bcdbdbacaba") || !strcmp(x,"bcdbdbacabaa") || !strcmp(x,"bcdbdbacabab") || !strcmp(x,"bcdbdbacababb") || !strcmp(x,"bcdbdbacababba") || !strcmp(x,"bcdbdbacababbb") || !strcmp(x,"bcdbdbacababbd") || !strcmp(x,"bcdbdbacababaa") || !strcmp(x,"bcdbdbacababab") || !strcmp(x,"bcdbdbacababac") || !strcmp(x,"bcdbdbacababad") || !strcmp(x,"bcdbdbacabaaa") || !strcmp(x,"bcdbdbacabaaba") || !strcmp(x,"bcdbdbacabaabb") || !strcmp(x,"bcdbdbacabaabc") || !strcmp(x,"bcdbdbacabaabd") || !strcmp(x,"bcdbdbacabaaaa") || !strcmp(x,"bcdbdbacabaaab") || !strcmp(x,"bcdbdbacabaaac") || !strcmp(x,"bcdbdbacabaaad") || 0)
  51 #endif
  52
  53
  54 static GHashTable *dedupe_ways_hash;
  55
  56 static int attr_debug_level=1;
  57 static int nodeid,wayid;
  58 static int report,phase;
  59 static int ignore_unkown = 0, coverage=0;
  60
  61 static char *attrmap={
  62         "n      *=*                     point_unkn\n"
  63         "n      amenity=hospital        poi_hospital\n"
  64         "n      amenity=atm             poi_bank\n"
  65         "n      amenity=bank            poi_bank\n"
  66         "n      amenity=pub             poi_bar\n"
  67         "n      amenity=cafe            poi_cafe\n"
  68         "n      amenity=bus_station     poi_bus_station\n"
  69         "n      amenity=parking         poi_car_parking\n"
  70         "n      amenity=cinema          poi_cinema\n"
  71         "n      amenity=fire_station    poi_firebrigade\n"
  72         "n      amenity=fuel            poi_fuel\n"
  73         "n      amenity=courthouse      poi_justice\n"
  74         "n      amenity=library         poi_library\n"
  75         "n      amenity=pharmacy        poi_pharmacy\n"
  76         "n      amenity=place_of_worship,religion=christian     poi_church\n"
  77         "n      amenity=police          poi_police\n"
  78         "n      amenity=post_office     poi_post\n"
  79         "n      amenity=post_box        poi_post\n"
  80         "n      amenity=public_building poi_public_office\n"
  81         "n      amenity=restaurant      poi_restaurant\n"
  82         "n      amenity=fast_food       poi_fastfood\n"
  83         "n      amenity=toilets         poi_restroom\n"
  84         "n      amenity=school          poi_school\n"
  85         "n      amenity=university      poi_school\n"
  86         "n      amenity=college         poi_school\n"
  87         "n      amenity=telephone       poi_telephone\n"
  88         "n      amenity=theatre         poi_theater\n"
  89         "n      amenity=townhall                poi_townhall\n"
  90         "n      highway=bus_stop        poi_bus_stop\n"
  91         "n      highway=mini_roundabout mini_roundabout\n"
  92         "n      highway=motorway_junction       highway_exit\n"
  93         "n      highway=traffic_signals traffic_signals\n"
  94         "n      highway=turning_circle  turning_circle\n"
  95         "n      leisure=slipway         poi_boat_ramp\n"
  96         "n      leisure=fishing         poi_fish\n"
  97         "n      sport=golf      poi_golf\n"
  98         "n      leisure=golf_course     poi_golf\n"
  99         "n      leisure=marina          poi_marine\n"
 100         "n      leisure=sports_centre   poi_sport\n"
 101         "n      leisure=stadium         poi_stadium\n"
 102         "n      shop=supermarket        poi_shopping\n"
 103         "n      shop=convenience        poi_shop_grocery\n"
 104         "n      tourism=attraction      poi_attraction\n"
 105         "n      tourism=camp_site       poi_camp_rv\n"
 106         "n      tourism=caravan_site    poi_camp_rv\n"
 107         "n      tourism=hotel           poi_hotel\n"
 108         "n      tourism=motel           poi_hotel\n"
 109         "n      tourism=guest_house     poi_hotel\n"
 110         "n      tourism=hostel          poi_hotel\n"
 111         "n      tourism=information     poi_information\n"
 112         "n      tourism=museum          poi_museum_history\n"
 113         "n      tourism=picnic_site     poi_picnic\n"
 114         "n      tourism=theme_park      poi_resort\n"
 115         "n      tourism=zoo             poi_zoo\n"
 116         "n      amenity=grave_yard      poi_cemetery\n"
 117         "n      landuse=cemetery        poi_cemetery\n"
 118         "n      military=airfield       poi_military\n"
 119         "n      military=bunker         poi_military\n"
 120         "n      military=barracks       poi_military\n"
 121         "n      military=range          poi_military\n"
 122         "n      military=danger_area    poi_danger_area\n"
 123         "n      sport=swimming  poi_swimming\n"
 124         "n      sport=skiing            poi_skiing\n"
 125         "n      aeroway=aerodrome       poi_airport\n"
 126         "n      aeroway=airport         poi_airport\n"
 127         "n      aeroway=terminal        poi_airport\n"
 128         "n      aeroway=helipad         poi_heliport\n"
 129         "n      man_made=tower  poi_tower\n"
 130         "n      natural=bay             poi_bay\n"
 131         "n      natural=peak            poi_peak\n"
 132         "n      place=suburb            district_label\n"
 133         "n      place=city              town_label_2e5\n"
 134         "n      place=town              town_label_2e4\n"
 135         "n      place=village           town_label_2e3\n"
 136         "n      place=hamlet            town_label_2e2\n"
 137         "n      place=locality          town_label_2e0\n"
 138         "n      railway=halt            poi_rail_halt\n"
 139         "n      railway=level_crossing          poi_level_crossing\n"
 140         "n      railway=station         poi_rail_station\n"
 141         "n      railway=tram_stop               poi_rail_tram_stop\n"
 142         "w      *=*                     street_unkn\n"
 143         "w      amenity=place_of_worship        poly_building\n"
 144         "w      building=*      poly_building\n"
 145         "w      aeroway=aerodrome       poly_airport\n"
 146         "w      aeroway=apron           poly_apron\n"
 147         "w      aeroway=runway          aeroway_runway\n"
 148         "w      aeroway=taxiway         aeroway_taxiway\n"
 149         "w      aeroway=terminal        poly_terminal\n"
 150         "w      highway=bridleway       bridleway\n"
 151         "w      highway=cycleway        cycleway\n"
 152         "w      highway=cyclepath       cycleway\n"
 153         "w      highway=footway         footway\n"
 154         "w      highway=steps           steps\n"
 155         "w      highway=unsurfaced      track_gravelled\n"
 156         "w      highway=track           track_gravelled\n"
 157         "w      highway=track,tracktype=grade1          track_paved\n"
 158         "w      highway=track,surface=paved             track_paved\n"
 159         "w      highway=track,tracktype=grade2          track_gravelled\n"
 160         "w      highway=track,surface=gravel            track_gravelled\n"
 161         "w      highway=track,tracktype=grade3          track_unpaved\n"
 162         "w      highway=track,surface=unpaved           track_unpaved\n"
 163         "w      highway=track,tracktype=grade4          track_ground\n"
 164         "w      highway=track,surface=ground            track_ground\n"
 165         "w      highway=track,tracktype=grade5          track_grass\n"
 166         "w      highway=track,surface=grass             track_grass\n"
 167         "w      highway=service         street_service\n"
 168         "w      highway=construction            street_construction\n"
 169         "w      highway=pedestrian      street_pedestrian\n"
 170         "w      highway=pedestrian,area=1       poly_pedestrian\n"
 171         "w      highway=plaza   poly_plaza\n"
 172         "w      landuse=plaza   poly_plaza\n"
 173         "w      highway=residential     street_1_city\n"
 174         "w      highway=living_street           living_street\n"
 175         "w      highway=unclassified    street_1_city\n"
 176         "w      highway=road    street_1_city\n"
 177         "w      highway=minor           street_1_land\n"
 178         "w      highway=tertiary        street_2_city\n"
 179         "w      highway=tertiary_link   ramp\n"
 180         "w      highway=secondary       street_3_city\n"
 181         "w      highway=secondary_link  ramp\n"
 182         "w      highway=primary         street_4_city\n"
 183         "w      highway=primary_link    ramp\n"
 184         "w      highway=trunk           street_4_city\n"
 185         "w      highway=trunk_link      ramp\n"
 186         "w      highway=motorway        highway_city\n"
 187         "w      highway=motorway_link   ramp\n"
 188         "w      historic=town gate      poly_building\n"
 189         "w      cycleway=track          cycleway\n"
 190         "w      amenity=parking         poly_car_parking\n"
 191         "w      landuse=allotments      poly_allotments\n"
 192         "w      landuse=cemetery        poly_cemetery\n"
 193         "w      amenity=grave_yard      poly_cemetery\n"
 194         "w      landuse=forest          poly_wood\n"
 195         "w      landuse=industrial      poly_industry\n"
 196         "w      landuse=residential     poly_town\n"
 197         "w      landuse=residential,area=1      poly_town\n"
 198         "w      amenity=college poly_college\n"
 199         "w      amenity=university      poly_university\n"
 200         "w      tourism=camp_site       poly_camp_site\n"
 201         "w      tourism=caravan_site    poly_caravan_site\n"
 202         "w      tourism=picnic_site     poly_picnic_site\n"
 203         "w      tourism=theme_park      poly_theme_park\n"
 204         "w      tourism=attraction      poly_attraction\n"
 205         "w      tourism=zoo     poly_zoo\n"
 206         "w      tourism=artwork poly_artwork\n"
 207         "w      historic=archaeological_site    poly_archaeological_site\n"
 208         "w      historic=ruins  poly_ruins\n"
 209         "w      historic=battlefield    poly_battlefield\n"
 210         "w      landuse=quarry  poly_quarry\n"
 211         "w      landuse=landfill        poly_landfill\n"
 212         "w      landuse=retail  poly_retail\n"
 213         "w      landuse=commercial      poly_commercial\n"
 214         "w      landuse=brownfield      poly_brownfield\n"
 215         "w      landuse=greenfield      poly_greenfield\n"
 216         "w      landuse=construction    poly_construction\n"
 217         "w      landuse=railway poly_railway\n"
 218         "w      natural=glacier poly_glacier\n"
 219         "w      natural=scree   poly_scree\n"
 220         "w      natural=scrub   poly_scrub\n"
 221         "w      natural=fell    poly_fell\n"
 222         "w      natural=heath   poly_heath\n"
 223         "w      natural=marsh   poly_marsh\n"
 224         "w      natural=mud     poly_mud\n"
 225         "w      natural=beach   poly_beach\n"
 226         "w      natural=land    poly_land\n"
 227         "w      landuse=basin   poly_basin\n"
 228         "w      landuse=reservoir       poly_reservoir\n"
 229         "w      landuse=farm    poly_farm\n"
 230         "w      landuse=vineyard        poly_farm\n"
 231         "w      leisure=park            poly_park\n"
 232         "w      landuse=village_green           poly_village_green\n"
 233         "w      landuse=recreation_ground               poly_recreation_ground\n"
 234         "w      natural=wood            poly_wood\n"
 235         "w      natural=water           poly_water\n"
 236         "w      natural=coastline       water_line\n"
 237         "w      place=suburb            poly_town\n"
 238         "w      place=town              poly_town\n"
 239         "w      landuse=military        poly_military\n"
 240         "w      military=airfield       poly_airfield\n"
 241         "w      military=barracks       poly_barracks\n"
 242         "w      military=danger_area    poly_danger_area\n"
 243         "w      military=range  poly_range\n"
 244         "w      military=naval_base     poly_naval_base\n"
 245         "w      power=line      powerline\n"
 246         "w      railway=rail            rail\n"
 247         "w      railway=narrow_gauge    rail_narrow_gauge\n"
 248         "w      railway=light_rail              rail_light\n"
 249         "w      railway=subway          rail_subway\n"
 250         "w      railway=tram            rail_tram\n"
 251         "w      railway=monorail                rail_mono\n"
 252         "w      railway=preserved               rail_preserved\n"
 253         "w      railway=disused         rail_disused\n"
 254         "w      railway=abandoned               rail_abandoned\n"
 255         "w      aerialway=cable_car             lift_cable_car\n"
 256         "w      aerialway=chair_lift            lift_chair\n"
 257         "w      aerialway=drag_lift             lift_drag\n"
 258         "w      leisure=golf_course     poly_golf_course\n"
 259         "w      sport=*         poly_sport\n"
 260         "w      leisure=sports_centre           poly_sport\n"
 261         "w      leisure=stadium         poly_sports_stadium\n"
 262         "w      leisure=track           poly_sports_track\n"
 263         "w      leisure=pitch           poly_sports_pitch\n"
 264         "w      leisure=water_park              poly_water_park\n"
 265         "w      leisure=marina          poly_marina\n"
 266         "w      leisure=fishing         poly_fishing\n"
 267         "w      leisure=garden          poly_garden\n"
 268         "w      leisure=common          poly_common\n"
 269         "w      leisure=playground              poly_playground\n"
 270         "w      leisure=nature_reserve          poly_nature_reserve\n"
 271         "w      waterway=canal          water_canal\n"
 272         "w      waterway=river          water_river\n"
 273         "w      waterway=stream         water_stream\n"
 274         "w      waterway=drain          water_drain\n"
 275         "w      waterway=riverbank      poly_water\n"
 276         "w      boundary=administrative border_country\n"
 277         "w      boundary=civil  border_civil\n"
 278         "w      boundary=political      border_political\n"
 279         "w      boundary=national_park  border_national_park\n"
 280         "w      route=ferry             ferry\n"
 281         "w      highway=bus_guideway            bus_guideway\n"
 282 };
 283
 284 struct coord {
 285         int x;
 286         int y;
 287 } coord_buffer[65536];
 288
 289 #define IS_REF(c) ((c).x >= (1 << 30))
 290 #define REF(c) ((c).y)
 291 #define SET_REF(c,ref) do { (c).x = 1 << 30; (c).y = ref ; } while(0)
 292
 293 struct rect {
 294         struct coord l,h;
 295 };
 296
 297 static void bbox_extend(struct coord *c, struct rect *r);
 298
 299 #ifdef GENERATE_INDEX
 300
 301 static GHashTable *aux_tile_hash;
 302 GList *aux_tile_list;
 303
 304 struct country_table {
 305         int countryid;
 306         char *names;
 307         FILE *file;
 308         int size;
 309         int count;
 310         struct rect r;
 311 } country_table[] = {
 312         { 40,"Austria,Österreich,AUT"},
 313         {124,"Canada"},
 314         {276,"Germany,Deutschland,Bundesrepublik Deutschland"},
 315         {528,"Nederland,The Netherlands,Niederlande,NL"},
 316         {756,"Schweiz"},
 317         {840,"USA"}
 318 };
 319
 320 static GHashTable *country_table_hash;
 321 #endif
 322
 323 struct attr_mapping {
 324         enum item_type type;
 325         int attr_present_idx_count;
 326         int attr_present_idx[0];
 327 };
 328
 329 static struct attr_mapping **attr_mapping_node;
 330 static int attr_mapping_node_count;
 331 static struct attr_mapping **attr_mapping_way;
 332 static int attr_mapping_way_count;
 333
 334 static char *attr_present;
 335 static int attr_present_count;
 336 static GHashTable *attr_hash;
 337
 338
 339 static GHashTable *strings_hash = NULL;
 340
 341
 342 static char* string_hash_lookup( const char* key )
 343 {
 344         char* key_ptr = NULL;
 345
 346         if ( strings_hash == NULL ) {
 347                 strings_hash = g_hash_table_new(g_str_hash, g_str_equal);
 348         }
 349
 350         if ( ( key_ptr = g_hash_table_lookup(strings_hash, key )) == NULL ) {
 351                 key_ptr = g_strdup( key );
 352                 g_hash_table_insert(strings_hash, key_ptr,  (gpointer)key_ptr );
 353
 354         }
 355         return key_ptr;
 356 }
 357
 358 static void
 359 build_attrmap_line(char *line)
 360 {
 361         char *t=NULL,*kvl=NULL,*i=NULL,*p,*kv;
 362         struct attr_mapping ***attr_mapping_curr,*attr_mapping=g_malloc0(sizeof(struct attr_mapping));
 363         int idx,attr_mapping_count=0,*attr_mapping_curr_count;
 364         t=line;
 365         p=strchr(t,'\t');
 366         if (p) {
 367                 while (*p == '\t')
 368                         *p++='\0';
 369                 kvl=p;
 370                 p=strchr(kvl,'\t');
 371         }
 372         if (p) {
 373                 while (*p == '\t')
 374                         *p++='\0';
 375                 i=p;
 376         }
 377         if (t[0] == 'w') {
 378                 if (! i)
 379                         i="street_unkn";
 380                 attr_mapping_curr=&attr_mapping_way;
 381                 attr_mapping_curr_count=&attr_mapping_way_count;
 382         } else {
 383                 if (! i)
 384                         i="point_unkn";
 385                 attr_mapping_curr=&attr_mapping_node;
 386                 attr_mapping_curr_count=&attr_mapping_node_count;
 387         }
 388         attr_mapping->type=item_from_name(i);
 389         while ((kv=strtok(kvl, ","))) {
 390                 kvl=NULL;
 391                 if (!(idx=(int)g_hash_table_lookup(attr_hash, kv))) {
 392                         idx=attr_present_count++;
 393                         g_hash_table_insert(attr_hash, kv, (gpointer) idx);
 394                 }
 395                 attr_mapping=g_realloc(attr_mapping, sizeof(struct attr_mapping)+(attr_mapping_count+1)*sizeof(int));
 396                 attr_mapping->attr_present_idx[attr_mapping_count++]=idx;
 397                 attr_mapping->attr_present_idx_count=attr_mapping_count;
 398         }
 399         *attr_mapping_curr=g_realloc(*attr_mapping_curr, sizeof(**attr_mapping_curr)*(*attr_mapping_curr_count+1));
 400         (*attr_mapping_curr)[(*attr_mapping_curr_count)++]=attr_mapping;
 401 }
 402
 403 static void
 404 build_attrmap(char *map)
 405 {
 406         char *p;
 407         attr_hash=g_hash_table_new(g_str_hash, g_str_equal);
 408         attr_present_count=1;
 409         while (map) {
 410                 p=strchr(map,'\n');
 411                 if (p)
 412                         *p++='\0';
 413                 if (strlen(map))
 414                         build_attrmap_line(map);
 415                 map=p;
 416         }
 417         attr_present=g_malloc0(sizeof(*attr_present)*attr_present_count);
 418 }
 419
 420 #ifdef GENERATE_INDEX
 421 static void
 422 build_countrytable(void)
 423 {
 424         int i;
 425         char *names,*str,*tok;
 426         country_table_hash=g_hash_table_new(g_str_hash, g_str_equal);
 427         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
 428                 names=g_strdup(country_table[i].names);
 429                 str=names;
 430                 while ((tok=strtok(str, ","))) {
 431                         str=NULL;
 432                         g_hash_table_insert(country_table_hash, tok,  (gpointer)&country_table[i]);
 433                 }
 434         }
 435 }
 436 #endif
 437
 438
 439
 440 static int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
 441 static int in_way, in_node, in_relation;
 442
 443 static void
 444 sig_alrm(int sig)
 445 {
 446 #ifndef _WIN32
 447         signal(SIGALRM, sig_alrm);
 448 #endif
 449         alarm(30);
 450         fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
 451 }
 452
 453 struct item_bin {
 454         int len;
 455         enum item_type type;
 456         int clen;
 457 } item;
 458
 459 struct attr_bin {
 460         int len;
 461         enum attr_type type;
 462 };
 463
 464 struct attr_bin label_attr = {
 465         0, attr_label
 466 };
 467 char label_attr_buffer[BUFFER_SIZE];
 468
 469 struct attr_bin town_name_attr = {
 470         0, attr_town_name
 471 };
 472
 473 struct attr_bin street_name_attr = {
 474         0, attr_street_name
 475 };
 476
 477 struct attr_bin street_name_systematic_attr = {
 478         0, attr_street_name_systematic
 479 };
 480 char street_name_systematic_attr_buffer[BUFFER_SIZE];
 481
 482 struct attr_bin debug_attr = {
 483         0, attr_debug
 484 };
 485 char debug_attr_buffer[BUFFER_SIZE];
 486
 487 struct attr_bin flags_attr = {
 488         0, attr_flags
 489 };
 490 int flags_attr_value;
 491
 492 char is_in_buffer[BUFFER_SIZE];
 493
 494
 495 static void write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level);
 496
 497 static void
 498 pad_text_attr(struct attr_bin *a, char *buffer)
 499 {
 500         int l;
 501         if (buffer && buffer[0]) {
 502                 l=strlen(buffer)+1;
 503                 while (l % 4)
 504                         buffer[l++]='\0';
 505                 a->len=l/4+1;
 506         } else
 507                 a->len=0;
 508 }
 509
 510 static int
 511 xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
 512 {
 513         int len=strlen(attribute);
 514         char *pos,*i,s,attr[len+2];
 515         strcpy(attr, attribute);
 516         strcpy(attr+len, "=");
 517         pos=strstr(xml, attr);
 518         if (! pos)
 519                 return 0;
 520         pos+=len+1;
 521         s=*pos++;
 522         if (! s)
 523                 return 0;
 524         i=strchr(pos, s);
 525         if (! i)
 526                 return 0;
 527         if (i - pos > buffer_size) {
 528                 fprintf(stderr,"Buffer overflow %d vs %d\n", i-pos, buffer_size);
 529                 return 0;
 530         }
 531         strncpy(buffer, pos, i-pos);
 532         buffer[i-pos]='\0';
 533         return 1;
 534 }
 535
 536 static int node_is_tagged;
 537
 538 static void
 539 add_tag(char *k, char *v)
 540 {
 541         GHashTable *value_hash;
 542         enum item_type type;
 543         int idx,level=2;
 544         char buffer[BUFFER_SIZE*2+2];
 545         if (! strcmp(k,"ele"))
 546                 level=9;
 547         if (! strcmp(k,"time"))
 548                 level=9;
 549         if (! strcmp(k,"created_by"))
 550                 level=9;
 551         if (! strncmp(k,"tiger:",6) || !strcmp(k,"AND_nodes"))
 552                 level=9;
 553         if (! strcmp(k,"converted_by") || ! strcmp(k,"source"))
 554                 level=8;
 555         if (! strncmp(k,"osmarender:",11) || !strncmp(k,"svg:",4))
 556                 level=8;
 557         if (! strcmp(k,"layer"))
 558                 level=7;
 559         if (! strcasecmp(v,"true") || ! strcasecmp(v,"yes"))
 560                 v="1";
 561         if (! strcmp(k,"oneway")) {
 562                 if (!strcmp(v,"1")) {
 563                         flags_attr_value=AF_ONEWAY;
 564                         flags_attr.len=2;
 565                 }
 566                 if (! strcmp(v,"-1")) {
 567                         flags_attr_value=AF_ONEWAYREV;
 568                         flags_attr.len=2;
 569                 }
 570                 if (!in_way)
 571                         level=6;
 572                 else
 573                         level=5;
 574         }
 575         if (! strcmp(k,"junction")) {
 576                 if (! strcmp(v,"roundabout")) {
 577                         flags_attr_value=AF_ONEWAY;
 578                         flags_attr.len=2;
 579                 }
 580         }
 581         if (! strcmp(k,"maxspeed")) {
 582                 level=5;
 583         }
 584         if (! strcmp(k,"bicycle")) {
 585                 level=5;
 586         }
 587         if (! strcmp(k,"foot")) {
 588                 level=5;
 589         }
 590         if (! strcmp(k,"note"))
 591                 level=5;
 592         if (! strcmp(k,"name")) {
 593                 strcpy(label_attr_buffer, v);
 594                 pad_text_attr(&label_attr, label_attr_buffer);
 595                 level=5;
 596         }
 597         if (! strcmp(k,"ref")) {
 598                 if (in_way) {
 599                         strcpy(street_name_systematic_attr_buffer, v);
 600                         pad_text_attr(&street_name_systematic_attr, street_name_systematic_attr_buffer);
 601                 }
 602                 level=5;
 603         }
 604         if (! strcmp(k,"is_in")) {
 605                 strcpy(is_in_buffer, v);
 606                 level=5;
 607         }
 608         if (! strcmp(k,"gnis:ST_alpha")) {
 609                 /*      assume a gnis tag means it is part of the USA:
 610                         http://en.wikipedia.org/wiki/Geographic_Names_Information_System
 611                         many US towns do not have is_in tags
 612                 */
 613                 strcpy(is_in_buffer, "USA");
 614                 level=5;
 615         }
 616         if (! strcmp(k,"lanes")) {
 617                 level=5;
 618         }
 619         if (attr_debug_level >= level) {
 620                 int bytes_left = sizeof( debug_attr_buffer ) - strlen(debug_attr_buffer) - 1;
 621                 if ( bytes_left > 0 )
 622                 {
 623                         snprintf(debug_attr_buffer+strlen(debug_attr_buffer), bytes_left,  " %s=%s", k, v);
 624                         debug_attr_buffer[ sizeof( debug_attr_buffer ) -  1 ] = '\0';
 625                         node_is_tagged=1;
 626                 }
 627         }
 628         if (level < 6)
 629                 node_is_tagged=1;
 630         if (level >= 5)
 631                 return;
 632
 633         strcpy(buffer,"*=*");
 634         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 635                 attr_present[idx]=1;
 636
 637         sprintf(buffer,"%s=*", k);
 638         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 639                 attr_present[idx]=2;
 640
 641         sprintf(buffer,"*=%s", v);
 642         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 643                 attr_present[idx]=2;
 644
 645         sprintf(buffer,"%s=%s", k, v);
 646         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 647                 attr_present[idx]=4;
 648 }
 649
 650 struct entity {
 651         char *entity;
 652         char c;
 653 } entities[]= {
 654         {"&quot;",'"'},
 655         {"&apos;",'\''},
 656         {"&amp;",'&'},
 657         {"&lt;",'<'},
 658         {"&gt;",'>'},
 659 };
 660
 661 static void
 662 decode_entities(char *buffer)
 663 {
 664         char *pos=buffer;
 665         int i,len,found;
 666
 667         while ((pos=strchr(pos, '&'))) {
 668                 found=0;
 669                 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) {
 670                         len=strlen(entities[i].entity);
 671                         if (!strncmp(pos, entities[i].entity, len)) {
 672                                 *pos=entities[i].c;
 673                                 memmove(pos+1, pos+len, strlen(pos+len)+1);
 674                                 found=1;
 675                                 break;
 676                         }
 677                 }
 678                 pos++;
 679         }
 680 }
 681
 682 static int
 683 parse_tag(char *p)
 684 {
 685         char k_buffer[BUFFER_SIZE];
 686         char v_buffer[BUFFER_SIZE];
 687         if (!xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
 688                 return 0;
 689         if (!xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
 690                 return 0;
 691         decode_entities(v_buffer);
 692         add_tag(k_buffer, v_buffer);
 693         return 1;
 694 }
 695
 696
 697 struct buffer {
 698         int malloced_step;
 699         size_t malloced;
 700         unsigned char *base;
 701         size_t size;
 702 };
 703
 704 static struct tile_head {
 705         int num_subtiles;
 706         int total_size;
 707         char *name;
 708         char *zip_data;
 709         int total_size_used;
 710         int zipnum;
 711         int process;
 712         struct tile_head *next;
 713         // char subtiles[0];
 714 } *tile_head_root;
 715
 716
 717 int coord_count;
 718
 719 struct node_item {
 720         int id;
 721         char ref_node;
 722         char ref_way;
 723         char ref_ref;
 724         char dummy;
 725         struct coord c;
 726 };
 727
 728 static struct buffer node_buffer = {
 729         64*1024*1024,
 730 };
 731
 732
 733 static char** th_get_subtile( const struct tile_head* th, int idx )
 734 {
 735         char* subtile_ptr = NULL;
 736         subtile_ptr = (char*)th + sizeof( struct tile_head ) + idx * sizeof( char *);
 737         return (char**)subtile_ptr;
 738 }
 739
 740 static void
 741 extend_buffer(struct buffer *b)
 742 {
 743         b->malloced+=b->malloced_step;
 744         b->base=realloc(b->base, b->malloced);
 745         if (b->base == NULL) {
 746                 fprintf(stderr,"realloc of %d bytes failed\n",b->malloced);
 747                 exit(1);
 748         }
 749
 750 }
 751
 752 int nodeid_last;
 753 GHashTable *node_hash;
 754
 755 static void
 756 node_buffer_to_hash(void)
 757 {
 758         int i,count=node_buffer.size/sizeof(struct node_item);
 759         struct node_item *ni=(struct node_item *)node_buffer.base;
 760         for (i = 0 ; i < count ; i++)
 761                 g_hash_table_insert(node_hash, (gpointer)(ni[i].id), (gpointer)i);
 762 }
 763
 764 static struct node_item *ni;
 765
 766 static void
 767 add_node(int id, double lat, double lon)
 768 {
 769         if (node_buffer.size + sizeof(struct node_item) > node_buffer.malloced)
 770                 extend_buffer(&node_buffer);
 771         node_is_tagged=0;
 772         nodeid=id;
 773         item.type=type_point_unkn;
 774         label_attr.len=0;
 775         town_name_attr.len=0;
 776         debug_attr.len=0;
 777         is_in_buffer[0]='\0';
 778         sprintf(debug_attr_buffer,"nodeid=%d", nodeid);
 779         ni=(struct node_item *)(node_buffer.base+node_buffer.size);
 780         ni->id=id;
 781         ni->ref_node=0;
 782         ni->ref_way=0;
 783         ni->ref_ref=0;
 784         ni->dummy=0;
 785         ni->c.x=lon*6371000.0*M_PI/180;
 786         ni->c.y=log(tan(M_PI_4+lat*M_PI/360))*6371000.0;
 787         node_buffer.size+=sizeof(struct node_item);
 788         if (! node_hash) {
 789                 if (ni->id > nodeid_last) {
 790                         nodeid_last=ni->id;
 791                 } else {
 792                         fprintf(stderr,"INFO: Nodes out of sequence (new %d vs old %d), adding hash\n", ni->id, nodeid_last);
 793                         node_hash=g_hash_table_new(NULL, NULL);
 794                         node_buffer_to_hash();
 795                 }
 796         } else
 797                 if (!g_hash_table_lookup(node_hash, (gpointer)(ni->id)))
 798                         g_hash_table_insert(node_hash, (gpointer)(ni->id), (gpointer)(ni-(struct node_item *)node_buffer.base));
 799                 else {
 800                         node_buffer.size-=sizeof(struct node_item);
 801                         nodeid=0;
 802                 }
 803
 804 }
 805
 806 static int
 807 parse_node(char *p)
 808 {
 809         char id_buffer[BUFFER_SIZE];
 810         char lat_buffer[BUFFER_SIZE];
 811         char lon_buffer[BUFFER_SIZE];
 812         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 813                 return 0;
 814         if (!xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
 815                 return 0;
 816         if (!xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
 817                 return 0;
 818         add_node(atoi(id_buffer), atof(lat_buffer), atof(lon_buffer));
 819         return 1;
 820 }
 821
 822
 823 static struct node_item *
 824 node_item_get(int id)
 825 {
 826         struct node_item *ni=(struct node_item *)(node_buffer.base);
 827         int count=node_buffer.size/sizeof(struct node_item);
 828         int interval=count/4;
 829         int p=count/2;
 830         if (node_hash) {
 831                 int i;
 832                 i=(int)(g_hash_table_lookup(node_hash, (gpointer)id));
 833                 return ni+i;
 834         }
 835         while (ni[p].id != id) {
 836 #if 0
 837                 fprintf(stderr,"p=%d count=%d interval=%d id=%d ni[p].id=%d\n", p, count, interval, id, ni[p].id);
 838 #endif
 839                 if (ni[p].id < id) {
 840                         p+=interval;
 841                         if (interval == 1) {
 842                                 if (p >= count)
 843                                         return NULL;
 844                                 if (ni[p].id > id)
 845                                         return NULL;
 846                         } else {
 847                                 if (p >= count)
 848                                         p=count-1;
 849                         }
 850                 } else {
 851                         p-=interval;
 852                         if (interval == 1) {
 853                                 if (p < 0)
 854                                         return NULL;
 855                                 if (ni[p].id < id)
 856                                         return NULL;
 857                         } else {
 858                                 if (p < 0)
 859                                         p=0;
 860                         }
 861                 }
 862                 if (interval > 1)
 863                         interval/=2;
 864         }
 865
 866         return &ni[p];
 867 }
 868
 869 static void
 870 node_ref_way(int id)
 871 {
 872         struct node_item *ni=node_item_get(id);
 873         if (! ni) {
 874                 fprintf(stderr,"WARNING: node id %d not found\n", id);
 875                 return;
 876         }
 877         ni->ref_way++;
 878 }
 879
 880
 881 static void
 882 add_way(int id)
 883 {
 884         wayid=id;
 885         coord_count=0;
 886         item.type=type_street_unkn;
 887         label_attr.len=0;
 888         street_name_attr.len=0;
 889         street_name_systematic_attr.len=0;
 890         debug_attr.len=0;
 891         flags_attr.len=0;
 892         sprintf(debug_attr_buffer,"wayid=%d", wayid);
 893 }
 894
 895 static int
 896 parse_way(char *p)
 897 {
 898         char id_buffer[BUFFER_SIZE];
 899         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 900                 return 0;
 901         add_way(atoi(id_buffer));
 902         return 1;
 903 }
 904
 905 static int
 906 parse_relation(char *p)
 907 {
 908         debug_attr_buffer[0]='\0';
 909         return 1;
 910 }
 911
 912 static void
 913 write_attr(FILE *out, struct attr_bin *attr, void *buffer)
 914 {
 915         if (attr->len) {
 916                 fwrite(attr, sizeof(*attr), 1, out);
 917                 fwrite(buffer, (attr->len-1)*4, 1, out);
 918         }
 919 }
 920
 921 static int
 922 attr_longest_match(struct attr_mapping **mapping, int mapping_count, enum item_type *types, int types_count)
 923 {
 924         int i,j,longest=0,ret=0,sum,val;
 925         struct attr_mapping *curr;
 926         for (i = 0 ; i < mapping_count ; i++) {
 927                 sum=0;
 928                 curr=mapping[i];
 929                 for (j = 0 ; j < curr->attr_present_idx_count ; j++) {
 930                         val=attr_present[curr->attr_present_idx[j]];
 931                         if (val)
 932                                 sum+=val;
 933                         else {
 934                                 sum=-1;
 935                                 break;
 936                         }
 937                 }
 938                 if (sum > longest) {
 939                         longest=sum;
 940                         ret=0;
 941                 }
 942                 if (sum > 0 && sum == longest && ret < types_count)
 943                         types[ret++]=curr->type;
 944         }
 945         memset(attr_present, 0, sizeof(*attr_present)*attr_present_count);
 946         return ret;
 947 }
 948
 949 static void
 950 end_way(FILE *out)
 951 {
 952         int alen=0,count;
 953         enum item_type types[5];
 954
 955         if (! out)
 956                 return;
 957         if (dedupe_ways_hash) {
 958                 if (g_hash_table_lookup(dedupe_ways_hash, (gpointer)wayid))
 959                         return;
 960                 g_hash_table_insert(dedupe_ways_hash, (gpointer)wayid, (gpointer)1);
 961         }
 962         count=attr_longest_match(attr_mapping_way, attr_mapping_way_count, types, sizeof(types)/sizeof(enum item_type));
 963         pad_text_attr(&debug_attr, debug_attr_buffer);
 964         if (label_attr.len)
 965                 alen+=label_attr.len+1;
 966         if (street_name_systematic_attr.len)
 967                 alen+=street_name_systematic_attr.len+1;
 968         if (debug_attr.len)
 969                 alen+=debug_attr.len+1;
 970         if (flags_attr.len)
 971                 alen+=flags_attr.len+1;
 972         if (count)
 973                 item.type=types[0];
 974         else
 975                 item.type=type_street_unkn;
 976         if (coverage && item_is_street(item))
 977                 item.type=type_coverage;
 978         item.clen=coord_count*2;
 979         item.len=item.clen+2+alen;
 980         fwrite(&item, sizeof(item), 1, out);
 981         fwrite(coord_buffer, coord_count*sizeof(struct coord), 1, out);
 982         if (item_is_street(item)) {
 983                 street_name_attr.len=label_attr.len;
 984                 write_attr(out, &street_name_attr, label_attr_buffer);
 985         } else
 986                 write_attr(out, &label_attr, label_attr_buffer);
 987         write_attr(out, &street_name_systematic_attr, street_name_systematic_attr_buffer);
 988         write_attr(out, &debug_attr, debug_attr_buffer);
 989         write_attr(out, &flags_attr, &flags_attr_value);
 990 }
 991
 992 static void
 993 end_node(FILE *out)
 994 {
 995         int alen=0,conflict=0,count;
 996         enum item_type types[5];
 997         struct country_table *result=NULL, *lookup;
 998         if (!out || ! node_is_tagged || ! nodeid)
 999                 return;
1000         count=attr_longest_match(attr_mapping_node, attr_mapping_node_count, types, sizeof(types)/sizeof(enum item_type));
1001         pad_text_attr(&debug_attr, debug_attr_buffer);
1002         if (label_attr.len)
1003                 alen+=label_attr.len+1;
1004         if (debug_attr.len)
1005                 alen+=debug_attr.len+1;
1006         if (count)
1007                 item.type=types[0];
1008         else
1009                 item.type=type_point_unkn;
1010         item.clen=2;
1011         item.len=item.clen+2+alen;
1012         fwrite(&item, sizeof(item), 1, out);
1013         fwrite(&ni->c, 1*sizeof(struct coord), 1, out);
1014         if (item_is_town(item)) {
1015                 town_name_attr.len=label_attr.len;
1016                 write_attr(out, &town_name_attr, label_attr_buffer);
1017         } else
1018                 write_attr(out, &label_attr, label_attr_buffer);
1019         write_attr(out, &debug_attr, debug_attr_buffer);
1020 #ifdef GENERATE_INDEX
1021         if (item_is_town(item) && town_name_attr.len) {
1022                 char *tok,*buf=is_in_buffer;
1023                 while ((tok=strtok(buf, ","))) {
1024                         while (*tok==' ')
1025                                 tok++;
1026                         lookup=g_hash_table_lookup(country_table_hash,tok);
1027                         if (lookup) {
1028                                 if (result && result->countryid != lookup->countryid) {
1029                                         fprintf(stderr,"conflict for %s %s country %d vs %d\n", label_attr_buffer, debug_attr_buffer, lookup->countryid, result->countryid);
1030                                         conflict=1;
1031                                 } else
1032                                         result=lookup;
1033                         }
1034                         buf=NULL;
1035                 }
1036                 if (result && !conflict) {
1037                         if (!result->file) {
1038                                 char *name=g_strdup_printf("country_%d.bin.unsorted", result->countryid);
1039                                 result->file=fopen(name,"w");
1040                                 g_free(name);
1041                         }
1042                         if (result->file) {
1043                                 item.clen=2;
1044                                 item.len=item.clen+2+label_attr.len+1;
1045                                 fwrite(&item, sizeof(item), 1, result->file);
1046                                 fwrite(&ni->c, 1*sizeof(struct coord), 1, result->file);
1047                                 write_attr(result->file, &town_name_attr, label_attr_buffer);
1048                                 result->count++;
1049                                 result->size+=(item.clen+3+label_attr.len+1)*4;
1050                         }
1051
1052                 }
1053         }
1054 #endif
1055         processed_nodes_out++;
1056 }
1057
1058 static int
1059 sort_countries_compare(const void *p1, const void *p2)
1060 {
1061         struct item_bin *ib1=*((struct item_bin **)p1),*ib2=*((struct item_bin **)p2);
1062         struct attr_bin *attr1,*attr2;
1063         char *s1,*s2;
1064         assert(ib1->clen==2);
1065         assert(ib2->clen==2);
1066         attr1=(struct attr_bin *)((int *)(ib1+1)+ib1->clen);
1067         attr2=(struct attr_bin *)((int *)(ib2+1)+ib1->clen);
1068         assert(attr1->type == attr_town_name);
1069         assert(attr2->type == attr_town_name);
1070         s1=(char *)(attr1+1);
1071         s2=(char *)(attr2+1);
1072         return strcmp(s1, s2);
1073 #if 0
1074         fprintf(stderr,"sort_countries_compare p1=%p p2=%p %s %s\n",p1,p2,s1,s2);
1075 #endif
1076         return 0;
1077 }
1078
1079 #ifdef GENERATE_INDEX
1080 static void
1081 sort_countries(void)
1082 {
1083         int i,j;
1084         struct country_table *co;
1085         struct coord *c;
1086         struct item_bin *ib;
1087         FILE *f;
1088         char *p,*buffer,**idx,*name;
1089         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1090                 co=&country_table[i];
1091                 if (co->file) {
1092                         fclose(co->file);
1093                         co->file=NULL;
1094                 }
1095                 if (co->size) {
1096                         buffer=malloc(co->size);
1097                         assert(buffer != NULL);
1098                         idx=malloc(co->count*sizeof(void *));
1099                         assert(idx != NULL);
1100                         name=g_strdup_printf("country_%d.bin.unsorted", co->countryid);
1101                         f=fopen(name,"r");
1102                         assert(f != NULL);
1103                         fread(buffer, co->size, 1, f);
1104                         fclose(f);
1105                         unlink(name);
1106                         g_free(name);
1107                         p=buffer;
1108                         for (j = 0 ; j < co->count ; j++) {
1109                                 idx[j]=p;
1110                                 p+=(*((int *)p)+1)*4;
1111                         }
1112                         qsort(idx, co->count, sizeof(void *), sort_countries_compare);
1113                         name=g_strdup_printf("country_%d.bin", co->countryid);
1114                         f=fopen(name,"w");
1115                         for (j = 0 ; j < co->count ; j++) {
1116                                 ib=(struct item_bin *)(idx[j]);
1117                                 c=(struct coord *)(ib+1);
1118                                 fwrite(ib, (ib->len+1)*4, 1, f);
1119                                 if (j)
1120                                         bbox_extend(c, &co->r);
1121                                 else
1122                                         co->r.l=*c;
1123                                         co->r.h=*c;
1124                         }
1125                         fclose(f);
1126                 }
1127         }
1128 }
1129 #endif
1130
1131 static void
1132 add_nd(char *p, int ref)
1133 {
1134         int len;
1135         struct node_item *ni;
1136         ni=node_item_get(ref);
1137         if (ni) {
1138 #if 0
1139                 coord_buffer[coord_count++]=ni->c;
1140 #else
1141                 SET_REF(coord_buffer[coord_count], ref);
1142                 coord_count++;
1143 #endif
1144                 ni->ref_way++;
1145         } else {
1146                 len=strlen(p);
1147                 if (len > 0 && p[len-1]=='\n')
1148                         p[len-1]='\0';
1149                 fprintf(stderr,"WARNING: way %d: node %d not found (%s)\n",wayid,ref,p);
1150         }
1151         if (coord_count > 65536) {
1152                 fprintf(stderr,"ERROR: Overflow\n");
1153                 exit(1);
1154         }
1155 }
1156
1157 static int
1158 parse_nd(char *p)
1159 {
1160         char ref_buffer[BUFFER_SIZE];
1161         if (!xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
1162                 return 0;
1163         add_nd(p, atoi(ref_buffer));
1164         return 1;
1165 }
1166
1167
1168 static void
1169 save_buffer(char *filename, struct buffer *b)
1170 {
1171         FILE *f;
1172         f=fopen(filename,"wb+");
1173         fwrite(b->base, b->size, 1, f);
1174         fclose(f);
1175 }
1176
1177 static void
1178 load_buffer(char *filename, struct buffer *b)
1179 {
1180         FILE *f;
1181         if (b->base)
1182                 free(b->base);
1183         b->malloced=0;
1184         f=fopen(filename,"rb");
1185         fseek(f, 0, SEEK_END);
1186         b->size=b->malloced=ftell(f);
1187         fprintf(stderr,"reading %d bytes from %s\n", b->size, filename);
1188         fseek(f, 0, SEEK_SET);
1189         b->base=malloc(b->size);
1190         assert(b->base != NULL);
1191         fread(b->base, b->size, 1, f);
1192         fclose(f);
1193 }
1194
1195 static int
1196 phase1(FILE *in, FILE *out_ways, FILE *out_nodes)
1197 {
1198         int size=4096;
1199         char buffer[size];
1200         char *p;
1201         sig_alrm(0);
1202         while (fgets(buffer, size, in)) {
1203                 p=strchr(buffer,'<');
1204                 if (! p) {
1205                         fprintf(stderr,"WARNING: wrong line %s\n", buffer);
1206                         continue;
1207                 }
1208                 if (!strncmp(p, "<?xml ",6)) {
1209                 } else if (!strncmp(p, "<osm ",5)) {
1210                 } else if (!strncmp(p, "<bound ",7)) {
1211                 } else if (!strncmp(p, "<node ",6)) {
1212                         if (!parse_node(p))
1213                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1214                         in_node=1;
1215                         processed_nodes++;
1216                 } else if (!strncmp(p, "<tag ",5)) {
1217                         if (!parse_tag(p))
1218                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1219                 } else if (!strncmp(p, "<way ",5)) {
1220                         in_way=1;
1221                         if (!parse_way(p))
1222                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1223                         processed_ways++;
1224                 } else if (!strncmp(p, "<nd ",4)) {
1225                         if (!parse_nd(p))
1226                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1227                 } else if (!strncmp(p, "<relation ",10)) {
1228                         in_relation=1;
1229                         if (!parse_relation(p))
1230                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1231                         processed_relations++;
1232                 } else if (!strncmp(p, "<member ",8)) {
1233                 } else if (!strncmp(p, "</node>",7)) {
1234                         in_node=0;
1235                         end_node(out_nodes);
1236                 } else if (!strncmp(p, "</way>",6)) {
1237                         in_way=0;
1238                         end_way(out_ways);
1239                 } else if (!strncmp(p, "</relation>",11)) {
1240                         in_relation=0;
1241                 } else if (!strncmp(p, "</osm>",6)) {
1242                 } else {
1243                         fprintf(stderr,"WARNING: unknown tag in %s\n", buffer);
1244                 }
1245         }
1246         sig_alrm(0);
1247         alarm(0);
1248         return 1;
1249 }
1250
1251 #ifdef HAVE_POSTGRESQL
1252 static int
1253 phase1_db(char *dbstr, FILE *out_ways, FILE *out_nodes)
1254 {
1255         PGconn *conn;
1256         PGresult *res,*node,*way,*tag;
1257         int count,tagged,i,j,k;
1258         long min, max, id, tag_id, node_id;
1259         char query[256];
1260
1261         sig_alrm(0);
1262         conn=PQconnectdb(dbstr);
1263         if (! conn) {
1264                 fprintf(stderr,"Failed to connect to database with '%s'\n",dbstr);
1265                 exit(1);
1266         }
1267         res=PQexec(conn, "begin");
1268         if (! res) {
1269                 fprintf(stderr, "Cannot begin transaction: %s\n", PQerrorMessage(conn));
1270                 PQclear(res);
1271                 exit(1);
1272         }
1273         res=PQexec(conn, "set transaction isolation level serializable");
1274         if (! res) {
1275                 fprintf(stderr, "Cannot set isolation level: %s\n", PQerrorMessage(conn));
1276                 PQclear(res);
1277                 exit(1);
1278         }
1279         res=PQexec(conn, "declare node cursor for select id,x(coordinate),y(coordinate) from node order by id");
1280         if (! res) {
1281                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1282                 PQclear(res);
1283                 exit(1);
1284         }
1285         res=PQexec(conn, "declare way cursor for select id from way order by id");
1286         if (! res) {
1287                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1288                 PQclear(res);
1289                 exit(1);
1290         }
1291         res=PQexec(conn, "declare relation cursor for select id from relation order by id");
1292         if (! res) {
1293                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1294                 PQclear(res);
1295                 exit(1);
1296         }
1297         for (;;) {
1298                 node=PQexec(conn, "fetch 100000 from node");
1299                 if (! node) {
1300                         fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1301                         PQclear(node);
1302                         exit(1);
1303                 }
1304                 count=PQntuples(node);
1305                 if (! count)
1306                         break;
1307                 min=atol(PQgetvalue(node, 0, 0));
1308                 max=atol(PQgetvalue(node, count-1, 0));
1309                 sprintf(query,"select node_id,name,value from node_tag where node_id >= %ld and node_id <= %ld order by node_id", min, max);
1310                 tag=PQexec(conn, query);
1311                 if (! tag) {
1312                         fprintf(stderr, "Cannot query node_tag: %s\n", PQerrorMessage(conn));
1313                         exit(1);
1314                 }
1315                 j=0;
1316                 for (i = 0 ; i < count ; i++) {
1317                         id=atol(PQgetvalue(node, i, 0));
1318                         add_node(id, atof(PQgetvalue(node, i, 1)), atof(PQgetvalue(node, i, 2)));
1319                         tagged=0;
1320                         in_node=1;
1321                         processed_nodes++;
1322                         while (j < PQntuples(tag)) {
1323                                 tag_id=atol(PQgetvalue(tag, j, 0));
1324                                 if (tag_id == id) {
1325                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1326                                         tagged=1;
1327                                         j++;
1328                                 }
1329                                 if (tag_id < id)
1330                                         j++;
1331                                 if (tag_id > id)
1332                                         break;
1333                         }
1334                         if (tagged)
1335                                 end_node(out_nodes);
1336                         in_node=0;
1337                 }
1338                 PQclear(tag);
1339                 PQclear(node);
1340         }
1341         for (;;) {
1342                 way=PQexec(conn, "fetch 100000 from way");
1343                 if (! way) {
1344                         fprintf(stderr, "Cannot setup cursor for ways: %s\n", PQerrorMessage(conn));
1345                         PQclear(node);
1346                         exit(1);
1347                 }
1348                 count=PQntuples(way);
1349                 if (! count)
1350                         break;
1351                 min=atol(PQgetvalue(way, 0, 0));
1352                 max=atol(PQgetvalue(way, count-1, 0));
1353                 sprintf(query,"select way_id,node_id from way_node where way_id >= %ld and way_id <= %ld order by way_id,sequence_id", min, max);
1354                 node=PQexec(conn, query);
1355                 if (! node) {
1356                         fprintf(stderr, "Cannot query way_node: %s\n", PQerrorMessage(conn));
1357                         exit(1);
1358                 }
1359                 sprintf(query,"select way_id,name,value from way_tag where way_id >= %ld and way_id <= %ld order by way_id", min, max);
1360                 tag=PQexec(conn, query);
1361                 if (! tag) {
1362                         fprintf(stderr, "Cannot query way_tag: %s\n", PQerrorMessage(conn));
1363                         exit(1);
1364                 }
1365                 j=0;
1366                 k=0;
1367                 for (i = 0 ; i < count ; i++) {
1368                         id=atol(PQgetvalue(way, i, 0));
1369                         add_way(id);
1370                         tagged=0;
1371                         in_way=1;
1372                         processed_ways++;
1373                         while (k < PQntuples(node)) {
1374                                 node_id=atol(PQgetvalue(node, k, 0));
1375                                 if (node_id == id) {
1376                                         add_nd("",atol(PQgetvalue(node, k, 1)));
1377                                         tagged=1;
1378                                         k++;
1379                                 }
1380                                 if (node_id < id)
1381                                         k++;
1382                                 if (node_id > id)
1383                                         break;
1384                         }
1385                         while (j < PQntuples(tag)) {
1386                                 tag_id=atol(PQgetvalue(tag, j, 0));
1387                                 if (tag_id == id) {
1388                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1389                                         tagged=1;
1390                                         j++;
1391                                 }
1392                                 if (tag_id < id)
1393                                         j++;
1394                                 if (tag_id > id)
1395                                         break;
1396                         }
1397                         if (tagged)
1398                                 end_way(out_ways);
1399                         in_way=0;
1400                 }
1401                 PQclear(tag);
1402                 PQclear(node);
1403                 PQclear(way);
1404         }
1405
1406         res=PQexec(conn, "commit");
1407         if (! res) {
1408                 fprintf(stderr, "Cannot commit transaction: %s\n", PQerrorMessage(conn));
1409                 PQclear(res);
1410                 exit(1);
1411         }
1412         sig_alrm(0);
1413         alarm(0);
1414         return 1;
1415 }
1416 #endif
1417
1418 static char buffer[150000];
1419
1420 int bytes_read=0;
1421
1422 static struct item_bin *
1423 read_item(FILE *in)
1424 {
1425         struct item_bin *ib=(struct item_bin *) buffer;
1426         int r,s;
1427         r=fread(ib, sizeof(*ib), 1, in);
1428         if (r != 1)
1429                 return NULL;
1430         bytes_read+=r;
1431         assert((ib->len+1) < sizeof(buffer));
1432         s=(ib->len+1)*4-sizeof(*ib);
1433         r=fread(ib+1, s, 1, in);
1434         if (r != 1)
1435                 return NULL;
1436         bytes_read+=r;
1437         return ib;
1438 }
1439
1440 static void
1441 bbox_extend(struct coord *c, struct rect *r)
1442 {
1443         if (c->x < r->l.x)
1444                 r->l.x=c->x;
1445         if (c->y < r->l.y)
1446                 r->l.y=c->y;
1447         if (c->x > r->h.x)
1448                 r->h.x=c->x;
1449         if (c->y > r->h.y)
1450                 r->h.y=c->y;
1451 }
1452
1453 static void
1454 bbox(struct coord *c, int count, struct rect *r)
1455 {
1456         if (! count)
1457                 return;
1458         r->l=*c;
1459         r->h=*c;
1460         while (--count) {
1461                 c++;
1462                 bbox_extend(c, r);
1463         }
1464 }
1465
1466 static int
1467 contains_bbox(int xl, int yl, int xh, int yh, struct rect *r)
1468 {
1469         if (r->h.x < xl || r->h.x > xh) {
1470                 return 0;
1471         }
1472         if (r->l.x > xh || r->l.x < xl) {
1473                 return 0;
1474         }
1475         if (r->h.y < yl || r->h.y > yh) {
1476                 return 0;
1477         }
1478         if (r->l.y > yh || r->l.y < yl) {
1479                 return 0;
1480         }
1481         return 1;
1482
1483 }
1484 struct rect world_bbox = {
1485         { -20000000, -20000000},
1486         {  20000000,  20000000},
1487 };
1488
1489 static void
1490 tile(struct rect *r, char *ret, int max)
1491 {
1492         int x0,x1,x2,x3,x4;
1493         int y0,y1,y2,y3,y4;
1494         int i;
1495         x0=world_bbox.l.x;
1496         y0=world_bbox.l.y;
1497         x4=world_bbox.h.x;
1498         y4=world_bbox.h.y;
1499         for (i = 0 ; i < max ; i++) {
1500                 x2=(x0+x4)/2;
1501                 y2=(y0+y4)/2;
1502                 x1=(x0+x2)/2;
1503                 y1=(y0+y2)/2;
1504                 x3=(x2+x4)/2;
1505                 y3=(y2+y4)/2;
1506                 if (     contains_bbox(x0,y0,x2,y2,r)) {
1507                         strcat(ret,"d");
1508                         x4=x2;
1509                         y4=y2;
1510                 } else if (contains_bbox(x2,y0,x4,y2,r)) {
1511                         strcat(ret,"c");
1512                         x0=x2;
1513                         y4=y2;
1514                 } else if (contains_bbox(x0,y2,x2,y4,r)) {
1515                         strcat(ret,"b");
1516                         x4=x2;
1517                         y0=y2;
1518                 } else if (contains_bbox(x2,y2,x4,y4,r)) {
1519                         strcat(ret,"a");
1520                         x0=x2;
1521                         y0=y2;
1522                 } else
1523                         return;
1524         }
1525 }
1526
1527 static void
1528 tile_bbox(char *tile, struct rect *r)
1529 {
1530         struct coord c;
1531         *r=world_bbox;
1532         while (*tile) {
1533                 c.x=(r->l.x+r->h.x)/2;
1534                 c.y=(r->l.y+r->h.y)/2;
1535                 switch (*tile) {
1536                 case 'a':
1537                         r->l.x=c.x;
1538                         r->l.y=c.y;
1539                         break;
1540                 case 'b':
1541                         r->h.x=c.x;
1542                         r->l.y=c.y;
1543                         break;
1544                 case 'c':
1545                         r->l.x=c.x;
1546                         r->h.y=c.y;
1547                         break;
1548                 case 'd':
1549                         r->h.x=c.x;
1550                         r->h.y=c.y;
1551                         break;
1552                 }
1553                 tile++;
1554         }
1555 }
1556
1557 GHashTable *tile_hash;
1558 GHashTable *tile_hash2;
1559
1560 static void
1561 tile_extend(char *tile, struct item_bin *ib, GList **tiles_list)
1562 {
1563         struct tile_head *th=NULL;
1564         if (debug_tile(tile))
1565                 fprintf(stderr,"Tile:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1566         if (tile_hash2)
1567                 th=g_hash_table_lookup(tile_hash2, tile);
1568         if (!th)
1569                 th=g_hash_table_lookup(tile_hash, tile);
1570         if (! th) {
1571                 th=malloc(sizeof(struct tile_head)+ sizeof( char* ) );
1572                 assert(th != NULL);
1573                 // strcpy(th->subtiles, tile);
1574                 th->num_subtiles=1;
1575                 th->total_size=0;
1576                 th->total_size_used=0;
1577                 th->zipnum=0;
1578                 th->zip_data=NULL;
1579                 th->name=string_hash_lookup(tile);
1580                 *th_get_subtile( th, 0 ) = th->name;
1581
1582                 if (tile_hash2)
1583                         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1584                 if (tiles_list)
1585                         *tiles_list=g_list_append(*tiles_list, string_hash_lookup( th->name ) );
1586                 processed_tiles++;
1587                 if (debug_tile(tile))
1588                         fprintf(stderr,"new '%s'\n", tile);
1589         }
1590         th->total_size+=ib->len*4+4;
1591         if (debug_tile(tile))
1592                 fprintf(stderr,"New total size of %s(%p):%d\n", th->name, th, th->total_size);
1593         g_hash_table_insert(tile_hash, string_hash_lookup( th->name ), th);
1594 }
1595
1596 static int
1597 tile_data_size(char *tile)
1598 {
1599         struct tile_head *th;
1600         th=g_hash_table_lookup(tile_hash, tile);
1601         if (! th)
1602                 return 0;
1603         return th->total_size;
1604 }
1605
1606 static int
1607 merge_tile(char *base, char *sub)
1608 {
1609         struct tile_head *thb, *ths;
1610         thb=g_hash_table_lookup(tile_hash, base);
1611         ths=g_hash_table_lookup(tile_hash, sub);
1612         if (! ths)
1613                 return 0;
1614         if (debug_tile(base) || debug_tile(sub))
1615                 fprintf(stderr,"merging '%s'(%p) (%d) with '%s'(%p) (%d)\n", base, thb, thb ? thb->total_size : 0, sub, ths, ths->total_size);
1616         if (! thb) {
1617                 thb=ths;
1618                 g_hash_table_remove(tile_hash, sub);
1619                 thb->name=string_hash_lookup(base);
1620                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1621
1622         } else {
1623                 thb=realloc(thb, sizeof(struct tile_head)+( ths->num_subtiles+thb->num_subtiles ) * sizeof( char*) );
1624                 assert(thb != NULL);
1625                 memcpy( th_get_subtile( thb, thb->num_subtiles ), th_get_subtile( ths, 0 ), ths->num_subtiles * sizeof( char*) );
1626                 thb->num_subtiles+=ths->num_subtiles;
1627                 thb->total_size+=ths->total_size;
1628                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1629                 g_hash_table_remove(tile_hash, sub);
1630                 g_free(ths);
1631         }
1632         return 1;
1633 }
1634
1635
1636 static void
1637 get_tiles_list_func(char *key, struct tile_head *th, GList **list)
1638 {
1639         *list=g_list_prepend(*list, key);
1640 }
1641
1642 static GList *
1643 get_tiles_list(void)
1644 {
1645         GList *ret=NULL;
1646         g_hash_table_foreach(tile_hash, (GHFunc)get_tiles_list_func, &ret);
1647         return ret;
1648 }
1649
1650 #if 0
1651 static void
1652 write_tile(char *key, struct tile_head *th, gpointer dummy)
1653 {
1654         FILE *f;
1655         char buffer[1024];
1656         fprintf(stderr,"DEBUG: Writing %s\n", key);
1657         strcpy(buffer,"tiles/");
1658         strcat(buffer,key);
1659 #if 0
1660         strcat(buffer,".bin");
1661 #endif
1662         f=fopen(buffer, "wb+");
1663         while (th) {
1664                 fwrite(th->data, th->size, 1, f);
1665                 th=th->next;
1666         }
1667         fclose(f);
1668 }
1669 #endif
1670
1671 static void
1672 write_item(char *tile, struct item_bin *ib)
1673 {
1674         struct tile_head *th;
1675         int size;
1676
1677         th=g_hash_table_lookup(tile_hash2, tile);
1678         if (! th)
1679                 th=g_hash_table_lookup(tile_hash, tile);
1680         if (th) {
1681                 if (th->process != 0 && th->process != 1) {
1682                         fprintf(stderr,"error with tile '%s' of length %d\n", tile, strlen(tile));
1683                         abort();
1684                 }
1685                 if (! th->process)
1686                         return;
1687                 if (debug_tile(tile))
1688                         fprintf(stderr,"Data:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1689                 size=(ib->len+1)*4;
1690                 if (th->total_size_used+size > th->total_size) {
1691                         fprintf(stderr,"Overflow in tile %s (used %d max %d item %d)\n", tile, th->total_size_used, th->total_size, size);
1692                         exit(1);
1693                         return;
1694                 }
1695                 memcpy(th->zip_data+th->total_size_used, ib, size);
1696                 th->total_size_used+=size;
1697         } else {
1698                 fprintf(stderr,"no tile hash found for %s\n", tile);
1699                 exit(1);
1700         }
1701 }
1702
1703
1704 static void
1705 write_item_part(FILE *out, struct item_bin *orig, int first, int last)
1706 {
1707         struct item_bin new;
1708         struct coord *c=(struct coord *)(orig+1);
1709         char *attr=(char *)(c+orig->clen/2);
1710         int attr_len=orig->len-orig->clen-2;
1711         processed_ways++;
1712         new.type=orig->type;
1713         new.clen=(last-first+1)*2;
1714         new.len=new.clen+attr_len+2;
1715 #if 0
1716         fprintf(stderr,"first %d last %d type 0x%x len %d clen %d attr_len %d\n", first, last, new.type, new.len, new.clen, attr_len);
1717 #endif
1718         fwrite(&new, sizeof(new), 1, out);
1719         fwrite(c+first, new.clen*4, 1, out);
1720         fwrite(attr, attr_len*4, 1, out);
1721 }
1722
1723 static int
1724 phase2(FILE *in, FILE *out)
1725 {
1726         struct coord *c;
1727         int i,ccount,last,ndref;
1728         struct item_bin *ib;
1729         struct node_item *ni;
1730
1731         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1732         sig_alrm(0);
1733         while ((ib=read_item(in))) {
1734 #if 0
1735                 fprintf(stderr,"type 0x%x len %d clen %d\n", ib->type, ib->len, ib->clen);
1736 #endif
1737                 ccount=ib->clen/2;
1738                 c=(struct coord *)(ib+1);
1739                 last=0;
1740                 for (i = 0 ; i < ccount ; i++) {
1741                         if (IS_REF(c[i])) {
1742                                 ndref=REF(c[i]);
1743                                 ni=node_item_get(ndref);
1744 #if 0
1745                                 fprintf(stderr,"ni=%p\n", ni);
1746 #endif
1747                                 c[i]=ni->c;
1748                                 if (ni->ref_way > 1 && i != 0 && i != ccount-1 && item_is_street(*ib)) {
1749                                         write_item_part(out, ib, last, i);
1750                                         last=i;
1751                                 }
1752                         }
1753                 }
1754                 write_item_part(out, ib, last, ccount-1);
1755         }
1756         sig_alrm(0);
1757         alarm(0);
1758         return 0;
1759 }
1760
1761 static void
1762 phase34_process_file(int phase, FILE *in)
1763 {
1764         struct item_bin *ib;
1765         struct rect r;
1766         char buffer[1024];
1767         int max;
1768
1769         while ((ib=read_item(in))) {
1770                 if (ib->type < 0x80000000)
1771                         processed_nodes++;
1772                 else
1773                         processed_ways++;
1774                 bbox((struct coord *)(ib+1), ib->clen/2, &r);
1775                 buffer[0]='\0';
1776                 max=14;
1777                 if (ib->type == type_street_n_lanes || ib->type == type_highway_city || ib->type == type_highway_land || ib->type == type_ramp)
1778                         max=8;
1779                 if (ib->type == type_street_3_city || ib->type == type_street_4_city || ib->type == type_street_3_land || ib->type == type_street_4_land)
1780                         max=12;
1781
1782                 tile(&r, buffer, max);
1783 #if 0
1784                 fprintf(stderr,"%s\n", buffer);
1785 #endif
1786                 if (phase == 3)
1787                         tile_extend(buffer, ib, NULL);
1788                 else
1789                         write_item(buffer, ib);
1790         }
1791 }
1792
1793 struct index_item {
1794         struct item_bin item;
1795         struct rect r;
1796         struct attr_bin attr_order_limit;
1797         short min;
1798         short max;
1799         struct attr_bin attr_zipfile_ref;
1800         int zipfile_ref;
1801 };
1802
1803 static void
1804 index_submap_add(int phase, struct tile_head *th, GList **tiles_list)
1805 {
1806         struct index_item ii;
1807         int len=strlen(th->name);
1808         char index_tile[len+1];
1809
1810         ii.min=(len > 4) ? len-4 : 0;
1811         ii.max=255;
1812         strcpy(index_tile, th->name);
1813         if (len > 6)
1814                 len=6;
1815         else
1816                 len=0;
1817         index_tile[len]=0;
1818         tile_bbox(th->name, &ii.r);
1819
1820         ii.item.len=sizeof(ii)/4-1;
1821         ii.item.type=type_submap;
1822         ii.item.clen=4;
1823
1824         ii.attr_order_limit.len=2;
1825         ii.attr_order_limit.type=attr_order_limit;
1826
1827         ii.attr_zipfile_ref.len=2;
1828         ii.attr_zipfile_ref.type=attr_zipfile_ref;
1829         ii.zipfile_ref=th->zipnum;
1830
1831         if (phase == 3)
1832                 tile_extend(index_tile, (struct item_bin *)&ii, tiles_list);
1833         else
1834                 write_item(index_tile, (struct item_bin *)&ii);
1835 #if 0
1836         unsigned int *c=(unsigned int *)&ii;
1837         int i;
1838         for (i = 0 ; i < sizeof(ii)/4 ; i++) {
1839                 fprintf(stderr,"%08x ", c[i]);
1840         }
1841         fprintf(stderr,"\n");
1842 #endif
1843 }
1844
1845 static int
1846 add_tile_hash(struct tile_head *th)
1847 {
1848         int idx,len,maxnamelen=0;
1849         char **data;
1850
1851 #if 0
1852         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1853 #endif
1854         for( idx = 0; idx < th->num_subtiles; idx++ ) {
1855
1856         data = th_get_subtile( th, idx );
1857
1858                 if (debug_tile(data) || debug_tile(th->name)) {
1859                         fprintf(stderr,"Parent for '%s' is '%s'\n", *data, th->name);
1860                 }
1861
1862                 g_hash_table_insert(tile_hash2, *data, th);
1863
1864                 len = strlen( *data );
1865
1866                 if (len > maxnamelen) {
1867                         maxnamelen=len;
1868                 }
1869         }
1870         return maxnamelen;
1871 }
1872
1873
1874 static int
1875 create_tile_hash(void)
1876 {
1877         struct tile_head *th;
1878         int len,maxnamelen=0;
1879
1880         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1881         th=tile_head_root;
1882         while (th) {
1883                 len=add_tile_hash(th);
1884                 if (len > maxnamelen)
1885                         maxnamelen=len;
1886                 th=th->next;
1887         }
1888         return maxnamelen;
1889 }
1890
1891 static void
1892 create_tile_hash_list(GList *list)
1893 {
1894         GList *next;
1895         struct tile_head *th;
1896
1897         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1898
1899         fprintf(stderr,"list=%p\n", list);
1900         next=g_list_first(list);
1901         while (next) {
1902                 th=g_hash_table_lookup(tile_hash, next->data);
1903                 if (!th) {
1904                         fprintf(stderr,"No tile found for '%s'\n", (char *)(next->data));
1905                 }
1906                 add_tile_hash(th);
1907                 next=g_list_next(next);
1908         }
1909 }
1910
1911 static void
1912 destroy_tile_hash(void)
1913 {
1914         g_hash_table_destroy(tile_hash2);
1915         tile_hash2=NULL;
1916 }
1917
1918 static int zipnum;
1919
1920 static void write_countrydir(int phase, int maxnamelen);
1921
1922 static void
1923 write_tilesdir(int phase, int maxlen, FILE *out)
1924 {
1925         int idx,len;
1926         GList *tiles_list,*next;
1927         char **data;
1928         struct tile_head *th,**last=NULL;
1929         zipnum=0;
1930
1931         tiles_list=get_tiles_list();
1932         if (phase == 3)
1933                 create_tile_hash_list(tiles_list);
1934         next=g_list_first(tiles_list);
1935         last=&tile_head_root;
1936         if (! maxlen) {
1937                 while (next) {
1938                         if (strlen(next->data) > maxlen)
1939                                 maxlen=strlen(next->data);
1940                         next=g_list_next(next);
1941                 }
1942         }
1943         len=maxlen;
1944         while (len >= 0) {
1945 #if 0
1946                 fprintf(stderr,"PROGRESS: collecting tiles with len=%d\n", len);
1947 #endif
1948 #ifdef GENERATE_INDEX
1949                 if (! len)
1950                         write_countrydir(phase, maxlen);
1951 #endif
1952                 next=g_list_first(tiles_list);
1953                 while (next) {
1954                         if (strlen(next->data) == len) {
1955                                 th=g_hash_table_lookup(tile_hash, next->data);
1956                                 if (phase == 3) {
1957                                         *last=th;
1958                                         last=&th->next;
1959                                         th->next=NULL;
1960                                         th->zipnum=zipnum;
1961                                         fprintf(out,"%s:%d",(char *)next->data,th->total_size);
1962
1963                                         for ( idx = 0; idx< th->num_subtiles; idx++ ){
1964                         data= th_get_subtile( th, idx );
1965                                                 fprintf(out,":%s", *data);
1966                                         }
1967
1968                                         fprintf(out,"\n");
1969                                 }
1970                                 if (th->name[0])
1971                                         index_submap_add(phase, th, &tiles_list);
1972                                 zipnum++;
1973                                 processed_tiles++;
1974                         }
1975                         next=g_list_next(next);
1976                 }
1977                 len--;
1978         }
1979 }
1980
1981 static void
1982 merge_tiles(void)
1983 {
1984         struct tile_head *th;
1985         char basetile[1024];
1986         char subtile[1024];
1987         GList *tiles_list_sorted,*last;
1988         int i,i_min,len,size_all,size[5],size_min,work_done;
1989         long long zip_size;
1990
1991         do {
1992                 tiles_list_sorted=get_tiles_list();
1993                 fprintf(stderr,"PROGRESS: sorting %d tiles\n", g_list_length(tiles_list_sorted));
1994                 tiles_list_sorted=g_list_sort(tiles_list_sorted, (GCompareFunc)strcmp);
1995                 fprintf(stderr,"PROGRESS: sorting %d tiles done\n", g_list_length(tiles_list_sorted));
1996                 last=g_list_last(tiles_list_sorted);
1997                 zip_size=0;
1998                 while (last) {
1999                         th=g_hash_table_lookup(tile_hash, last->data);
2000                         zip_size+=th->total_size;
2001                         last=g_list_previous(last);
2002                 }
2003                 fprintf(stderr,"DEBUG: size=%Ld\n", zip_size);
2004                 last=g_list_last(tiles_list_sorted);
2005                 work_done=0;
2006                 while (last) {
2007                         processed_tiles++;
2008                         len=strlen(last->data);
2009                         if (len >= 1) {
2010                                 strcpy(basetile,last->data);
2011                                 basetile[len-1]='\0';
2012                                 strcpy(subtile,last->data);
2013                                 for (i = 0 ; i < 4 ; i++) {
2014                                         subtile[len-1]='a'+i;
2015                                         size[i]=tile_data_size(subtile);
2016                                 }
2017                                 size[4]=tile_data_size(basetile);
2018                                 size_all=size[0]+size[1]+size[2]+size[3]+size[4];
2019                                 if (size_all < 65536 && size_all > 0 && size_all != size[4]) {
2020                                         for (i = 0 ; i < 4 ; i++) {
2021                                                 subtile[len-1]='a'+i;
2022                                                 work_done+=merge_tile(basetile, subtile);
2023                                         }
2024                                 } else {
2025                                         for (;;) {
2026                                                 size_min=size_all;
2027                                                 i_min=-1;
2028                                                 for (i = 0 ; i < 4 ; i++) {
2029                                                         if (size[i] && size[i] < size_min) {
2030                                                                 size_min=size[i];
2031                                                                 i_min=i;
2032                                                         }
2033                                                 }
2034                                                 if (i_min == -1)
2035                                                         break;
2036                                                 if (size[4]+size_min >= 65536)
2037                                                         break;
2038                                                 subtile[len-1]='a'+i_min;
2039                                                 work_done+=merge_tile(basetile, subtile);
2040                                                 size[4]+=size[i_min];
2041                                                 size[i_min]=0;
2042                                         }
2043                                 }
2044                         }
2045                         last=g_list_previous(last);
2046                 }
2047                 g_list_free(tiles_list_sorted);
2048                 fprintf(stderr,"PROGRESS: merged %d tiles\n", work_done);
2049         } while (work_done);
2050 }
2051
2052 struct country_index_item {
2053         struct item_bin item;
2054         struct attr_bin attr_country_id;
2055         int country_id;
2056         struct attr_bin attr_zipfile_ref;
2057         int zipfile_ref;
2058 };
2059
2060 static void
2061 index_country_add(int phase, int country_id, int zipnum)
2062 {
2063         struct country_index_item ii;
2064         char *index_tile="";
2065
2066         ii.item.len=sizeof(ii)/4-1;
2067         ii.item.type=type_countryindex;
2068         ii.item.clen=0;
2069
2070         ii.attr_country_id.len=2;
2071         ii.attr_country_id.type=attr_country_id;
2072         ii.country_id=country_id;
2073
2074         ii.attr_zipfile_ref.len=2;
2075         ii.attr_zipfile_ref.type=attr_zipfile_ref;
2076         ii.zipfile_ref=zipnum;
2077
2078         if (phase == 3)
2079                 tile_extend(index_tile, (struct item_bin *)&ii, NULL);
2080         else
2081                 write_item(index_tile, (struct item_bin *)&ii);
2082 }
2083
2084 #ifdef GENERATE_INDEX
2085 struct aux_tile {
2086         char *name;
2087         char *filename;
2088         int size;
2089 };
2090
2091 static int
2092 add_aux_tile(int phase, char *name, char *filename, int size)
2093 {
2094         struct aux_tile *at;
2095         if (phase == 3) {
2096                 at=g_new0(struct aux_tile, 1);
2097                 at->name=g_strdup(name);
2098                 at->filename=g_strdup(filename);
2099                 at->size=size;
2100                 aux_tile_list=g_list_append(aux_tile_list, at);
2101         }
2102         return zipnum++;
2103 }
2104
2105 static int
2106 write_aux_tiles(FILE *out, FILE *dir_out, int compression_level, int namelen)
2107 {
2108         GList *l=aux_tile_list;
2109         struct aux_tile *at;
2110         char *buffer;
2111         FILE *f;
2112         int count=0;
2113
2114         while (l) {
2115                 at=l->data;
2116                 buffer=malloc(at->size);
2117                 assert(buffer != NULL);
2118                 f=fopen(at->filename,"r");
2119                 assert(f != NULL);
2120                 fread(buffer, at->size, 1, f);
2121                 fclose(f);
2122                 write_zipmember(out, dir_out, at->name, namelen, buffer, at->size, compression_level);
2123                 count++;
2124                 l=g_list_next(l);
2125         }
2126         return count;
2127 }
2128
2129 static void
2130 write_countrydir(int phase, int maxnamelen)
2131 {
2132         int i,zipnum;
2133         int max=11;
2134         char tilename[32];
2135         char searchtile[32];
2136         char filename[32];
2137         struct country_table *co;
2138         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2139                 co=&country_table[i];
2140                 if (co->size) {
2141                         tilename[0]='\0';
2142                         tile(&co->r, tilename, max);
2143                         sprintf(searchtile,"%ss%d", tilename, 0);
2144                         sprintf(filename,"country_%d.bin", co->countryid);
2145                         zipnum=add_aux_tile(phase, searchtile, filename, co->size);
2146                         index_country_add(phase,co->countryid,zipnum);
2147                 }
2148         }
2149 }
2150
2151 static void
2152 remove_countryfiles(void)
2153 {
2154         int i;
2155         char filename[32];
2156         struct country_table *co;
2157
2158         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2159                 co=&country_table[i];
2160                 if (co->size) {
2161                         sprintf(filename,"country_%d.bin", co->countryid);
2162                         unlink(filename);
2163                 }
2164         }
2165 }
2166 #endif
2167
2168 static int
2169 phase34(int phase, int maxnamelen, FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2170 {
2171
2172         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
2173         bytes_read=0;
2174         sig_alrm(0);
2175         if (phase == 3)
2176                 tile_hash=g_hash_table_new(g_str_hash, g_str_equal);
2177         if (ways_in)
2178                 phase34_process_file(phase, ways_in);
2179         if (nodes_in)
2180                 phase34_process_file(phase, nodes_in);
2181         fprintf(stderr,"read %d bytes\n", bytes_read);
2182         if (phase == 3)
2183                 merge_tiles();
2184         sig_alrm(0);
2185         alarm(0);
2186         write_tilesdir(phase, maxnamelen, tilesdir_out);
2187
2188         return 0;
2189
2190 }
2191
2192 static int
2193 phase3(FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2194 {
2195         return phase34(3, 0, ways_in, nodes_in, tilesdir_out);
2196 }
2197
2198 static long long zipoffset;
2199 static int zipdir_size;
2200
2201 static int
2202 compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)
2203 {
2204         z_stream stream;
2205         int err;
2206
2207         stream.next_in = (Bytef*)source;
2208         stream.avail_in = (uInt)sourceLen;
2209         stream.next_out = dest;
2210         stream.avail_out = (uInt)*destLen;
2211         if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
2212
2213         stream.zalloc = (alloc_func)0;
2214         stream.zfree = (free_func)0;
2215         stream.opaque = (voidpf)0;
2216
2217         err = deflateInit2(&stream, level, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY);
2218         if (err != Z_OK) return err;
2219
2220         err = deflate(&stream, Z_FINISH);
2221         if (err != Z_STREAM_END) {
2222                 deflateEnd(&stream);
2223                 return err == Z_OK ? Z_BUF_ERROR : err;
2224         }
2225         *destLen = stream.total_out;
2226
2227         err = deflateEnd(&stream);
2228         return err;
2229 }
2230
2231 static void
2232 write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level)
2233 {
2234         struct zip_lfh lfh = {
2235                 0x04034b50,
2236                 0x0a,
2237                 0x0,
2238                 0x0,
2239                 0xbe2a,
2240                 0x5d37,
2241                 0x0,
2242                 0x0,
2243                 0x0,
2244                 filelen,
2245                 0x0,
2246         };
2247         struct zip_cd cd = {
2248                 0x02014b50,
2249                 0x17,
2250                 0x00,
2251                 0x0a,
2252                 0x00,
2253                 0x0000,
2254                 0x0,
2255                 0xbe2a,
2256                 0x5d37,
2257                 0x0,
2258                 0x0,
2259                 0x0,
2260                 filelen,
2261                 0x0000,
2262                 0x0000,
2263                 0x0000,
2264                 0x0000,
2265                 0x0,
2266                 zipoffset,
2267         };
2268         char filename[filelen+1];
2269         int error,crc,len,comp_size=data_size;
2270         uLongf destlen=data_size+data_size/500+12;
2271         char compbuffer[destlen];
2272
2273         crc=crc32(0, NULL, 0);
2274         crc=crc32(crc, (unsigned char *)data, data_size);
2275         if (compression_level) {
2276                 error=compress2_int((Byte *)compbuffer, &destlen, (Bytef *)data, data_size, compression_level);
2277                 if (error == Z_OK) {
2278                         if (destlen < data_size) {
2279                                 data=compbuffer;
2280                                 comp_size=destlen;
2281                         }
2282                 } else {
2283                         fprintf(stderr,"compress2 returned %d\n", error);
2284                 }
2285         }
2286         lfh.zipcrc=crc;
2287         lfh.zipsize=comp_size;
2288         lfh.zipuncmp=data_size;
2289         lfh.zipmthd=compression_level ? 8:0;
2290         cd.zipccrc=crc;
2291         cd.zipcsiz=comp_size;
2292         cd.zipcunc=data_size;
2293         cd.zipcmthd=compression_level ? 8:0;
2294         strcpy(filename, name);
2295         len=strlen(filename);
2296         while (len < filelen) {
2297                 filename[len++]='_';
2298         }
2299         filename[filelen]='\0';
2300         fwrite(&lfh, sizeof(lfh), 1, out);
2301         fwrite(filename, filelen, 1, out);
2302         fwrite(data, comp_size, 1, out);
2303         zipoffset+=sizeof(lfh)+filelen+comp_size;
2304         fwrite(&cd, sizeof(cd), 1, dir_out);
2305         fwrite(filename, filelen, 1, dir_out);
2306         zipdir_size+=sizeof(cd)+filelen;
2307 }
2308
2309 static int
2310 process_slice(FILE *ways_in, FILE *nodes_in, int size, int maxnamelen, FILE *out, FILE *dir_out, int compression_level)
2311 {
2312         struct tile_head *th;
2313         char *slice_data,*zip_data;
2314         int zipfiles=0;
2315
2316         slice_data=malloc(size);
2317         assert(slice_data != NULL);
2318         zip_data=slice_data;
2319         th=tile_head_root;
2320         while (th) {
2321                 if (th->process) {
2322                         th->zip_data=zip_data;
2323                         zip_data+=th->total_size;
2324                 }
2325                 th=th->next;
2326         }
2327         if (ways_in)
2328                 fseek(ways_in, 0, SEEK_SET);
2329         if (nodes_in)
2330                 fseek(nodes_in, 0, SEEK_SET);
2331         phase34(4, maxnamelen, ways_in, nodes_in, NULL);
2332
2333         th=tile_head_root;
2334         while (th) {
2335                 if (th->process) {
2336 #ifdef GENERATE_INDEX
2337                         if (! strlen(th->name))
2338                                 zipfiles+=write_aux_tiles(out, dir_out, compression_level, maxnamelen);
2339 #endif
2340                         if (th->total_size != th->total_size_used) {
2341                                 fprintf(stderr,"Size error '%s': %d vs %d\n", th->name, th->total_size, th->total_size_used);
2342                                 exit(1);
2343                         } else {
2344                                 if (strlen(th->name))
2345                                         write_zipmember(out, dir_out, th->name, maxnamelen, th->zip_data, th->total_size, compression_level);
2346                                 else {
2347                                         write_zipmember(out, dir_out, "index", sizeof("index")-1, th->zip_data, th->total_size, compression_level);
2348                                 }
2349                                 zipfiles++;
2350                         }
2351                 }
2352                 th=th->next;
2353         }
2354         free(slice_data);
2355
2356         return zipfiles;
2357 }
2358
2359 static void
2360 cat(FILE *in, FILE *out)
2361 {
2362         size_t size;
2363         char buffer[4096];
2364         while ((size=fread(buffer, 1, 4096, in)))
2365                 fwrite(buffer, 1, size, out);
2366 }
2367
2368 static int
2369 phase4(FILE *ways_in, FILE *nodes_in, FILE *out, FILE *dir_out, int compression_level)
2370 {
2371         int slice_size=1024*1024*1024;
2372         int maxnamelen,size,slices;
2373         int zipfiles=0;
2374         struct tile_head *th,*th2;
2375         struct zip_eoc eoc = {
2376                 0x06054b50,
2377                 0x0000,
2378                 0x0000,
2379                 0x0000,
2380                 0x0000,
2381                 0x0,
2382                 0x0,
2383                 0x0,
2384         };
2385
2386         maxnamelen=create_tile_hash();
2387
2388         th=tile_head_root;
2389         size=0;
2390         slices=0;
2391         fprintf(stderr, "Maximum slice size %d\n", slice_size);
2392         while (th) {
2393                 if (size + th->total_size > slice_size) {
2394                         fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2395                         size=0;
2396                         slices++;
2397                 }
2398                 size+=th->total_size;
2399                 th=th->next;
2400         }
2401         if (size)
2402                 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2403         th=tile_head_root;
2404         size=0;
2405         slices=0;
2406         while (th) {
2407                 th2=tile_head_root;
2408                 while (th2) {
2409                         th2->process=0;
2410                         th2=th2->next;
2411                 }
2412                 size=0;
2413                 while (th && size+th->total_size < slice_size) {
2414                         size+=th->total_size;
2415                         th->process=1;
2416                         th=th->next;
2417                 }
2418                 zipfiles+=process_slice(ways_in, nodes_in, size, maxnamelen, out, dir_out, compression_level);
2419                 slices++;
2420         }
2421         fseek(dir_out, 0, SEEK_SET);
2422         cat(dir_out, out);
2423         eoc.zipenum=zipfiles;
2424         eoc.zipecenn=zipfiles;
2425         eoc.zipecsz=zipdir_size;
2426         eoc.zipeofst=zipoffset;
2427         fwrite(&eoc, sizeof(eoc), 1, out);
2428         sig_alrm(0);
2429         alarm(0);
2430         return 0;
2431 }
2432
2433 static void
2434 usage(FILE *f)
2435 {
2436         fprintf(f,"\n");
2437         fprintf(f,"osm2navit - parse osm textfile and converts to NavIt binfile format\n\n");
2438         fprintf(f,"Usage :\n");
2439         fprintf(f,"bzcat planet.osm.bz2 | osm2navit mymap.bin\n");
2440         fprintf(f,"Available switches:\n");
2441         fprintf(f,"-h (--help)              : this screen\n");
2442         fprintf(f,"-N (--nodes-only)        : process only nodes\n");
2443         fprintf(f,"-W (--ways-only)         : process only ways\n");
2444         fprintf(f,"-a (--attr-debug-level)  : control which data is included in the debug attribute\n");
2445         fprintf(f,"-c (--dump-coordinates)  : dump coordinates after phase 1\n");
2446 #ifdef HAVE_POSTGRESQL
2447         fprintf(f,"-d (--db)                : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
2448 #endif
2449         fprintf(f,"-e (--end)               : end at specified phase\n");
2450         fprintf(f,"-k (--keep-tmpfiles)     : do not delete tmp files after processing. useful to reuse them\n\n");
2451         fprintf(f,"-o (--coverage)          : map every street to item overage\n");
2452         fprintf(f,"-s (--start)             : start at specified phase\n");
2453         fprintf(f,"-i (--input-file)        : specify the input file name (OSM), overrules default stdin\n");
2454         fprintf(f,"-w (--dedupe-ways)       : ensure no duplicate ways or nodes. useful when using several input files\n");
2455         fprintf(f,"-z (--compression-level) : set the compression level\n");
2456         exit(1);
2457 }
2458
2459 int main(int argc, char **argv)
2460 {
2461         FILE *ways=NULL,*ways_split=NULL,*nodes=NULL,*tilesdir,*zipdir,*res;
2462         char *map=g_strdup(attrmap);
2463         int c,start=1,end=4,dump_coordinates=0;
2464         int keep_tmpfiles=0;
2465         int process_nodes=1, process_ways=1;
2466         int compression_level=9;
2467         char *result,*dbstr=NULL;
2468         FILE* input_file = stdin;
2469
2470
2471         while (1) {
2472 #if 0
2473                 int this_option_optind = optind ? optind : 1;
2474 #endif
2475                 int option_index = 0;
2476                 static struct option long_options[] = {
2477                         {"attr-debug-level", 1, 0, 'a'},
2478                         {"compression-level", 1, 0, 'z'},
2479                         {"coverage", 0, 0, 'o'},
2480 #ifdef HAVE_POSTGRESQL
2481                         {"db", 1, 0, 'd'},
2482 #endif
2483                         {"dedupe-ways", 0, 0, 'w'},
2484                         {"end", 1, 0, 'e'},
2485                         {"help", 0, 0, 'h'},
2486                         {"keep-tmpfiles", 0, 0, 'k'},
2487                         {"nodes-only", 0, 0, 'N'},
2488                         {"start", 1, 0, 's'},
2489                         {"input-file", 1, 0, 'i'},
2490                         {"ignore-unknown", 0, 0, 'n'},
2491                         {"ways-only", 0, 0, 'W'},
2492                         {0, 0, 0, 0}
2493                 };
2494                 c = getopt_long (argc, argv, "Nni:Wa:c"
2495 #ifdef HAVE_POSTGRESQL
2496                                               "d:"
2497 #endif
2498                                               "e:hks:w", long_options, &option_index);
2499                 if (c == -1)
2500                         break;
2501                 switch (c) {
2502                 case 'N':
2503                         process_ways=0;
2504                         break;
2505                 case 'W':
2506                         process_nodes=0;
2507                         break;
2508                 case 'a':
2509                         attr_debug_level=atoi(optarg);
2510                         break;
2511                 case 'c':
2512                         dump_coordinates=1;
2513                         break;
2514 #ifdef HAVE_POSTGRESQL
2515                 case 'd':
2516                         dbstr=optarg;
2517                         break;
2518 #endif
2519                 case 'e':
2520                         end=atoi(optarg);
2521                         break;
2522                 case 'h':
2523                         usage(stdout);
2524                         break;
2525                 case 'n':
2526                         fprintf(stderr,"I will IGNORE unknown types\n");
2527                         ignore_unkown=1;
2528                         break;
2529                 case 'k':
2530                         fprintf(stderr,"I will KEEP tmp files\n");
2531                         keep_tmpfiles=1;
2532                         break;
2533                 case 'o':
2534                         coverage=1;
2535                         break;
2536                 case 's':
2537                         start=atoi(optarg);
2538                         break;
2539                 case 'w':
2540                         dedupe_ways_hash=g_hash_table_new(NULL, NULL);
2541                         break;
2542                 case 'i':
2543                         input_file = fopen( optarg, "r" );
2544                         if ( input_file ==  NULL )
2545                         {
2546                             fprintf( stderr, "\nInput file (%s) not found\n", optarg );
2547                             exit( -1 );
2548                         }
2549                         break;
2550                 case 'z':
2551                         compression_level=atoi(optarg);
2552                         break;
2553                 case '?':
2554                         usage(stderr);
2555                         break;
2556                 default:
2557                         fprintf(stderr,"c=%d\n", c);
2558                 }
2559
2560         }
2561         if (optind != argc-1)
2562                 usage(stderr);
2563         result=argv[optind];
2564         build_attrmap(map);
2565 #ifdef GENERATE_INDEX
2566         build_countrytable();
2567 #endif
2568
2569
2570         if (start == 1) {
2571                 if (process_ways)
2572                         ways=fopen("ways.tmp","wb+");
2573                 if (process_nodes)
2574                         nodes=fopen("nodes.tmp","wb+");
2575                 phase=1;
2576                 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
2577 #ifdef HAVE_POSTGRESQL
2578                 if (dbstr)
2579                         phase1_db(dbstr,ways,nodes);
2580                 else
2581 #endif
2582                         phase1(input_file,ways,nodes);
2583                 if (ways)
2584                         fclose(ways);
2585                 if (nodes)
2586                         fclose(nodes);
2587 #ifdef GENERATE_INDEX
2588                 fprintf(stderr,"PROGRESS: Phase 1: sorting countries\n");
2589                 sort_countries();
2590 #endif
2591         }
2592         if (end == 1 || dump_coordinates)
2593                 save_buffer("coords.tmp",&node_buffer);
2594         if (end == 1)
2595                 exit(0);
2596         if (start == 2)
2597                 load_buffer("coords.tmp",&node_buffer);
2598         if (start <= 2) {
2599                 if (process_ways) {
2600                         ways=fopen("ways.tmp","rb");
2601                         ways_split=fopen("ways_split.tmp","wb+");
2602                         phase=2;
2603                         fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
2604                         phase2(ways,ways_split);
2605                         fclose(ways_split);
2606                         fclose(ways);
2607                         if(!keep_tmpfiles)
2608                                 remove("ways.tmp");
2609                 } else
2610                         fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
2611         }
2612         free(node_buffer.base);
2613         node_buffer.base=NULL;
2614         node_buffer.malloced=0;
2615         node_buffer.size=0;
2616         if (end == 2)
2617                 exit(0);
2618         if (start <= 3) {
2619                 phase=3;
2620                 fprintf(stderr,"PROGRESS: Phase 3: generating tiles\n");
2621                 if (process_ways)
2622                         ways_split=fopen("ways_split.tmp","rb");
2623                 if (process_nodes)
2624                         nodes=fopen("nodes.tmp","rb");
2625                 tilesdir=fopen("tilesdir.tmp","wb+");
2626                 phase3(ways_split,nodes,tilesdir);
2627                 fclose(tilesdir);
2628                 if (nodes)
2629                         fclose(nodes);
2630                 if (ways_split)
2631                         fclose(ways_split);
2632         }
2633         if (end == 3)
2634                 exit(0);
2635         if (start <= 4) {
2636                 phase=4;
2637                 fprintf(stderr,"PROGRESS: Phase 4: assembling map\n");
2638                 if (process_ways)
2639                         ways_split=fopen("ways_split.tmp","rb");
2640                 if (process_nodes)
2641                         nodes=fopen("nodes.tmp","rb");
2642                 res=fopen(result,"wb+");
2643                 zipdir=fopen("zipdir.tmp","wb+");
2644                 phase4(ways_split,nodes,res,zipdir,compression_level);
2645                 fclose(zipdir);
2646                 fclose(res);
2647                 if (nodes)
2648                         fclose(nodes);
2649                 if (ways_split)
2650                         fclose(ways_split);
2651                 if(!keep_tmpfiles) {
2652                         remove("nodes.tmp");
2653                         remove("ways_split.tmp");
2654                         remove("tilesdir.tmp");
2655                         remove("zipdir.tmp");
2656 #ifdef GENERATE_INDEX
2657                         remove_countryfiles();
2658 #endif
2659                 }
2660         }
2661         return 0;
2662 }