vcs.maemo.org Git - navit-package/blob - navit/osm2navit.c

   1 /**
   2  * Navit, a modular navigation system.
   3  * Copyright (C) 2005-2008 Navit Team
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the
  16  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  17  * Boston, MA  02110-1301, USA.
  18  */
  19
  20
  21 #define _FILE_OFFSET_BITS 64
  22 #define _LARGEFILE_SOURCE
  23 #define _LARGEFILE64_SOURCE
  24 #include <glib.h>
  25 #include <assert.h>
  26 #include <string.h>
  27 #include <signal.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <math.h>
  31 #include <getopt.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <sys/stat.h>
  35 #include <zlib.h>
  36 #include "item.h"
  37 #include "zipfile.h"
  38 #include "config.h"
  39
  40 #define BUFFER_SIZE 1280
  41
  42 #define GENERATE_INDEX
  43
  44 #if 1
  45 #define debug_tile(x) 0
  46 #else
  47 #define debug_tile(x) (!strcmp(x,"bcdbd") || !strcmp(x,"bcdbd") || !strcmp(x,"bcdbda") || !strcmp(x,"bcdbdb") || !strcmp(x,"bcdbdba") || !strcmp(x,"bcdbdbb") || !strcmp(x,"bcdbdbba") || !strcmp(x,"bcdbdbaa") || !strcmp(x,"bcdbdbacaa") || !strcmp(x,"bcdbdbacab") || !strcmp(x,"bcdbdbacaba") || !strcmp(x,"bcdbdbacabaa") || !strcmp(x,"bcdbdbacabab") || !strcmp(x,"bcdbdbacababb") || !strcmp(x,"bcdbdbacababba") || !strcmp(x,"bcdbdbacababbb") || !strcmp(x,"bcdbdbacababbd") || !strcmp(x,"bcdbdbacababaa") || !strcmp(x,"bcdbdbacababab") || !strcmp(x,"bcdbdbacababac") || !strcmp(x,"bcdbdbacababad") || !strcmp(x,"bcdbdbacabaaa") || !strcmp(x,"bcdbdbacabaaba") || !strcmp(x,"bcdbdbacabaabb") || !strcmp(x,"bcdbdbacabaabc") || !strcmp(x,"bcdbdbacabaabd") || !strcmp(x,"bcdbdbacabaaaa") || !strcmp(x,"bcdbdbacabaaab") || !strcmp(x,"bcdbdbacabaaac") || !strcmp(x,"bcdbdbacabaaad") || 0)
  48 #endif
  49
  50 #define IS_TOWN(item) ((item).type >= type_town_label && (item).type <= type_town_label_1e7)
  51 #define IS_STREET(item) ((item).type >= type_street_nopass && (item).type <= type_ferry)
  52
  53 static GHashTable *dedupe_ways_hash;
  54
  55 static int attr_debug_level=1;
  56 static int nodeid,wayid;
  57 static int report,phase;
  58 static int ignore_unkown = 0, coverage=0;
  59
  60 static char *attrmap={
  61         "n      *=*                     point_unkn\n"
  62         "n      amenity=hospital        poi_hospital\n"
  63         "n      amenity=atm             poi_bank\n"
  64         "n      amenity=bank            poi_bank\n"
  65         "n      amenity=pub             poi_bar\n"
  66         "n      amenity=cafe            poi_cafe\n"
  67         "n      amenity=bus_station     poi_bus_station\n"
  68         "n      amenity=parking         poi_car_parking\n"
  69         "n      amenity=cinema          poi_cinema\n"
  70         "n      amenity=fire_station    poi_firebrigade\n"
  71         "n      amenity=fuel            poi_fuel\n"
  72         "n      amenity=courthouse      poi_justice\n"
  73         "n      amenity=library         poi_library\n"
  74         "n      amenity=pharmacy        poi_pharmacy\n"
  75         "n      amenity=place_of_worship,religion=christian     poi_church\n"
  76         "n      amenity=police          poi_police\n"
  77         "n      amenity=post_office     poi_post\n"
  78         "n      amenity=post_box        poi_post\n"
  79         "n      amenity=public_building poi_public_office\n"
  80         "n      amenity=restaurant      poi_restaurant\n"
  81         "n      amenity=fast_food       poi_fastfood\n"
  82         "n      amenity=toilets         poi_restroom\n"
  83         "n      amenity=school          poi_school\n"
  84         "n      amenity=university      poi_school\n"
  85         "n      amenity=college         poi_school\n"
  86         "n      amenity=telephone       poi_telephone\n"
  87         "n      amenity=theatre         poi_theater\n"
  88         "n      amenity=townhall                poi_townhall\n"
  89         "n      highway=bus_stop        poi_bus_stop\n"
  90         "n      highway=mini_roundabout mini_roundabout\n"
  91         "n      highway=motorway_junction       highway_exit\n"
  92         "n      highway=traffic_signals traffic_signals\n"
  93         "n      highway=turning_circle  turning_circle\n"
  94         "n      leisure=slipway         poi_boat_ramp\n"
  95         "n      leisure=fishing         poi_fish\n"
  96         "n      sport=golf      poi_golf\n"
  97         "n      leisure=golf_course     poi_golf\n"
  98         "n      leisure=marina          poi_marine\n"
  99         "n      leisure=sports_centre   poi_sport\n"
 100         "n      leisure=stadium         poi_stadium\n"
 101         "n      shop=supermarket        poi_shopping\n"
 102         "n      shop=convenience        poi_shop_grocery\n"
 103         "n      tourism=attraction      poi_attraction\n"
 104         "n      tourism=camp_site       poi_camp_rv\n"
 105         "n      tourism=caravan_site    poi_camp_rv\n"
 106         "n      tourism=hotel           poi_hotel\n"
 107         "n      tourism=motel           poi_hotel\n"
 108         "n      tourism=guest_house     poi_hotel\n"
 109         "n      tourism=hostel          poi_hotel\n"
 110         "n      tourism=information     poi_information\n"
 111         "n      tourism=museum          poi_museum_history\n"
 112         "n      tourism=picnic_site     poi_picnic\n"
 113         "n      tourism=theme_park      poi_resort\n"
 114         "n      tourism=zoo             poi_zoo\n"
 115         "n      amenity=grave_yard      poi_cemetery\n"
 116         "n      landuse=cemetery        poi_cemetery\n"
 117         "n      military=airfield       poi_military\n"
 118         "n      military=bunker         poi_military\n"
 119         "n      military=barracks       poi_military\n"
 120         "n      military=range          poi_military\n"
 121         "n      military=danger_area    poi_danger_area\n"
 122         "n      sport=swimming  poi_swimming\n"
 123         "n      sport=skiing            poi_skiing\n"
 124         "n      aeroway=aerodrome       poi_airport\n"
 125         "n      aeroway=airport         poi_airport\n"
 126         "n      aeroway=terminal        poi_airport\n"
 127         "n      aeroway=helipad         poi_heliport\n"
 128         "n      man_made=tower  poi_tower\n"
 129         "n      natural=bay             poi_bay\n"
 130         "n      natural=peak            poi_peak\n"
 131         "n      place=suburb            district_label\n"
 132         "n      place=city              town_label_2e5\n"
 133         "n      place=town              town_label_2e4\n"
 134         "n      place=village           town_label_2e3\n"
 135         "n      place=hamlet            town_label_2e2\n"
 136         "n      place=locality          town_label_2e0\n"
 137         "n      railway=halt            poi_rail_halt\n"
 138         "n      railway=level_crossing          poi_level_crossing\n"
 139         "n      railway=station         poi_rail_station\n"
 140         "n      railway=tram_stop               poi_rail_tram_stop\n"
 141         "w      *=*                     street_unkn\n"
 142         "w      amenity=place_of_worship        poly_building\n"
 143         "w      building=*      poly_building\n"
 144         "w      aeroway=aerodrome       poly_airport\n"
 145         "w      aeroway=apron           poly_apron\n"
 146         "w      aeroway=runway          aeroway_runway\n"
 147         "w      aeroway=taxiway         aeroway_taxiway\n"
 148         "w      aeroway=terminal        poly_terminal\n"
 149         "w      highway=bridleway       bridleway\n"
 150         "w      highway=cycleway        cycleway\n"
 151         "w      highway=cyclepath       cycleway\n"
 152         "w      highway=footway         footway\n"
 153         "w      highway=steps           steps\n"
 154         "w      highway=unsurfaced      track_gravelled\n"
 155         "w      highway=track           track_gravelled\n"
 156         "w      highway=track,tracktype=grade1          track_paved\n"
 157         "w      highway=track,surface=paved             track_paved\n"
 158         "w      highway=track,tracktype=grade2          track_gravelled\n"
 159         "w      highway=track,surface=gravel            track_gravelled\n"
 160         "w      highway=track,tracktype=grade3          track_unpaved\n"
 161         "w      highway=track,surface=unpaved           track_unpaved\n"
 162         "w      highway=track,tracktype=grade4          track_ground\n"
 163         "w      highway=track,surface=ground            track_ground\n"
 164         "w      highway=track,tracktype=grade5          track_grass\n"
 165         "w      highway=track,surface=grass             track_grass\n"
 166         "w      highway=service         street_service\n"
 167         "w      highway=construction            street_construction\n"
 168         "w      highway=pedestrian      street_pedestrian\n"
 169         "w      highway=pedestrian,area=1       poly_pedestrian\n"
 170         "w      highway=plaza   poly_plaza\n"
 171         "w      landuse=plaza   poly_plaza\n"
 172         "w      highway=residential     street_1_city\n"
 173         "w      highway=living_street           living_street\n"
 174         "w      highway=unclassified    street_1_city\n"
 175         "w      highway=road    street_1_city\n"
 176         "w      highway=minor           street_1_land\n"
 177         "w      highway=tertiary        street_2_city\n"
 178         "w      highway=secondary       street_3_city\n"
 179         "w      highway=primary         street_4_city\n"
 180         "w      highway=primary_link    ramp\n"
 181         "w      highway=trunk           street_4_city\n"
 182         "w      highway=trunk_link      ramp\n"
 183         "w      highway=motorway        highway_city\n"
 184         "w      highway=motorway_link   ramp\n"
 185         "w      historic=town gate      poly_building\n"
 186         "w      cycleway=track          cycleway\n"
 187         "w      amenity=parking         poly_car_parking\n"
 188         "w      landuse=allotments      poly_allotments\n"
 189         "w      landuse=cemetery        poly_cemetery\n"
 190         "w      amenity=grave_yard      poly_cemetery\n"
 191         "w      landuse=forest          poly_wood\n"
 192         "w      landuse=industrial      poly_industry\n"
 193         "w      landuse=residential     poly_town\n"
 194         "w      landuse=residential,area=1      poly_town\n"
 195         "w      amenity=college poly_college\n"
 196         "w      amenity=university      poly_university\n"
 197         "w      tourism=camp_site       poly_camp_site\n"
 198         "w      tourism=caravan_site    poly_caravan_site\n"
 199         "w      tourism=picnic_site     poly_picnic_site\n"
 200         "w      tourism=theme_park      poly_theme_park\n"
 201         "w      tourism=attraction      poly_attraction\n"
 202         "w      tourism=zoo     poly_zoo\n"
 203         "w      tourism=artwork poly_artwork\n"
 204         "w      historic=archaeological_site    poly_archaeological_site\n"
 205         "w      historic=ruins  poly_ruins\n"
 206         "w      historic=battlefield    poly_battlefield\n"
 207         "w      landuse=quarry  poly_quarry\n"
 208         "w      landuse=landfill        poly_landfill\n"
 209         "w      landuse=retail  poly_retail\n"
 210         "w      landuse=commercial      poly_commercial\n"
 211         "w      landuse=brownfield      poly_brownfield\n"
 212         "w      landuse=greenfield      poly_greenfield\n"
 213         "w      landuse=construction    poly_construction\n"
 214         "w      landuse=railway poly_railway\n"
 215         "w      natural=glacier poly_glacier\n"
 216         "w      natural=scree   poly_scree\n"
 217         "w      natural=scrub   poly_scrub\n"
 218         "w      natural=fell    poly_fell\n"
 219         "w      natural=heath   poly_heath\n"
 220         "w      natural=marsh   poly_marsh\n"
 221         "w      natural=mud     poly_mud\n"
 222         "w      natural=beach   poly_beach\n"
 223         "w      natural=land    poly_land\n"
 224         "w      landuse=basin   poly_basin\n"
 225         "w      landuse=reservoir       poly_reservoir\n"
 226         "w      landuse=farm    poly_farm\n"
 227         "w      landuse=vineyard        poly_farm\n"
 228         "w      leisure=park            poly_park\n"
 229         "w      landuse=village_green           poly_village_green\n"
 230         "w      landuse=recreation_ground               poly_recreation_ground\n"
 231         "w      natural=wood            poly_wood\n"
 232         "w      natural=water           poly_water\n"
 233         "w      natural=coastline       water_line\n"
 234         "w      place=suburb            poly_town\n"
 235         "w      place=town              poly_town\n"
 236         "w      landuse=military        poly_military\n"
 237         "w      military=airfield       poly_airfield\n"
 238         "w      military=barracks       poly_barracks\n"
 239         "w      military=danger_area    poly_danger_area\n"
 240         "w      military=range  poly_range\n"
 241         "w      military=naval_base     poly_naval_base\n"
 242         "w      power=line      powerline\n"
 243         "w      railway=rail            rail\n"
 244         "w      railway=narrow_gauge    rail_narrow_gauge\n"
 245         "w      railway=light_rail              rail_light\n"
 246         "w      railway=subway          rail_subway\n"
 247         "w      railway=tram            rail_tram\n"
 248         "w      railway=monorail                rail_mono\n"
 249         "w      railway=preserved               rail_preserved\n"
 250         "w      railway=disused         rail_disused\n"
 251         "w      railway=abandoned               rail_abandoned\n"
 252         "w      aerialway=cable_car             lift_cable_car\n"
 253         "w      aerialway=chair_lift            lift_chair\n"
 254         "w      aerialway=drag_lift             lift_drag\n"
 255         "w      leisure=golf_course     poly_golf_course\n"
 256         "w      sport=*         poly_sport\n"
 257         "w      leisure=sports_centre           poly_sport\n"
 258         "w      leisure=stadium         poly_sports_stadium\n"
 259         "w      leisure=track           poly_sports_track\n"
 260         "w      leisure=pitch           poly_sports_pitch\n"
 261         "w      leisure=water_park              poly_water_park\n"
 262         "w      leisure=marina          poly_marina\n"
 263         "w      leisure=fishing         poly_fishing\n"
 264         "w      leisure=garden          poly_garden\n"
 265         "w      leisure=common          poly_common\n"
 266         "w      leisure=playground              poly_playground\n"
 267         "w      leisure=nature_reserve          poly_nature_reserve\n"
 268         "w      waterway=canal          water_canal\n"
 269         "w      waterway=river          water_river\n"
 270         "w      waterway=stream         water_stream\n"
 271         "w      waterway=drain          water_drain\n"
 272         "w      waterway=riverbank      poly_water\n"
 273         "w      boundary=administrative border_country\n"
 274         "w      boundary=civil  border_civil\n"
 275         "w      boundary=political      border_political\n"
 276         "w      boundary=national_park  border_national_park\n"
 277         "w      route=ferry             ferry\n"
 278         "w      highway=bus_guideway            bus_guideway\n"
 279 };
 280
 281 struct coord {
 282         int x;
 283         int y;
 284 } coord_buffer[65536];
 285
 286 #define IS_REF(c) ((c).x >= (1 << 30))
 287 #define REF(c) ((c).y)
 288 #define SET_REF(c,ref) do { (c).x = 1 << 30; (c).y = ref ; } while(0)
 289
 290 struct rect {
 291         struct coord l,h;
 292 };
 293
 294 static void bbox_extend(struct coord *c, struct rect *r);
 295
 296 #ifdef GENERATE_INDEX
 297
 298 static GHashTable *aux_tile_hash;
 299 GList *aux_tile_list;
 300
 301 struct country_table {
 302         int countryid;
 303         char *names;
 304         FILE *file;
 305         int size;
 306         int count;
 307         struct rect r;
 308 } country_table[] = {
 309         { 40,"Austria,Österreich,AUT"},
 310         {124,"Canada"},
 311         {276,"Germany,Deutschland,Bundesrepublik Deutschland"},
 312         {528,"Nederland,The Netherlands,Niederlande,NL"},
 313         {756,"Schweiz"},
 314 };
 315
 316 static GHashTable *country_table_hash;
 317 #endif
 318
 319 struct attr_mapping {
 320         enum item_type type;
 321         int attr_present_idx_count;
 322         int attr_present_idx[0];
 323 };
 324
 325 static struct attr_mapping **attr_mapping_node;
 326 static int attr_mapping_node_count;
 327 static struct attr_mapping **attr_mapping_way;
 328 static int attr_mapping_way_count;
 329
 330 static char *attr_present;
 331 static int attr_present_count;
 332 static GHashTable *attr_hash;
 333
 334
 335 static GHashTable *strings_hash = NULL;
 336
 337
 338 static char* string_hash_lookup( const char* key )
 339 {
 340         char* key_ptr = NULL;
 341
 342         if ( strings_hash == NULL ) {
 343                 strings_hash = g_hash_table_new(g_str_hash, g_str_equal);
 344         }
 345
 346         if ( ( key_ptr = g_hash_table_lookup(strings_hash, key )) == NULL ) {
 347                 key_ptr = g_strdup( key );
 348                 g_hash_table_insert(strings_hash, key_ptr,  (gpointer)key_ptr );
 349
 350         }
 351         return key_ptr;
 352 }
 353
 354 static void
 355 build_attrmap_line(char *line)
 356 {
 357         char *t=NULL,*kvl=NULL,*i=NULL,*p,*kv;
 358         struct attr_mapping ***attr_mapping_curr,*attr_mapping=g_malloc0(sizeof(struct attr_mapping));
 359         int idx,attr_mapping_count=0,*attr_mapping_curr_count;
 360         t=line;
 361         p=strchr(t,'\t');
 362         if (p) {
 363                 while (*p == '\t')
 364                         *p++='\0';
 365                 kvl=p;
 366                 p=strchr(kvl,'\t');
 367         }
 368         if (p) {
 369                 while (*p == '\t')
 370                         *p++='\0';
 371                 i=p;
 372         }
 373         if (t[0] == 'w') {
 374                 if (! i)
 375                         i="street_unkn";
 376                 attr_mapping_curr=&attr_mapping_way;
 377                 attr_mapping_curr_count=&attr_mapping_way_count;
 378         } else {
 379                 if (! i)
 380                         i="point_unkn";
 381                 attr_mapping_curr=&attr_mapping_node;
 382                 attr_mapping_curr_count=&attr_mapping_node_count;
 383         }
 384         attr_mapping->type=item_from_name(i);
 385         while ((kv=strtok(kvl, ","))) {
 386                 kvl=NULL;
 387                 if (!(idx=(int)g_hash_table_lookup(attr_hash, kv))) {
 388                         idx=attr_present_count++;
 389                         g_hash_table_insert(attr_hash, kv, (gpointer) idx);
 390                 }
 391                 attr_mapping=g_realloc(attr_mapping, sizeof(struct attr_mapping)+(attr_mapping_count+1)*sizeof(int));
 392                 attr_mapping->attr_present_idx[attr_mapping_count++]=idx;
 393                 attr_mapping->attr_present_idx_count=attr_mapping_count;
 394         }
 395         *attr_mapping_curr=g_realloc(*attr_mapping_curr, sizeof(**attr_mapping_curr)*(*attr_mapping_curr_count+1));
 396         (*attr_mapping_curr)[(*attr_mapping_curr_count)++]=attr_mapping;
 397 }
 398
 399 static void
 400 build_attrmap(char *map)
 401 {
 402         char *p;
 403         attr_hash=g_hash_table_new(g_str_hash, g_str_equal);
 404         attr_present_count=1;
 405         while (map) {
 406                 p=strchr(map,'\n');
 407                 if (p)
 408                         *p++='\0';
 409                 if (strlen(map))
 410                         build_attrmap_line(map);
 411                 map=p;
 412         }
 413         attr_present=g_malloc0(sizeof(*attr_present)*attr_present_count);
 414 }
 415
 416 #ifdef GENERATE_INDEX
 417 static void
 418 build_countrytable(void)
 419 {
 420         int i;
 421         char *names,*str,*tok;
 422         country_table_hash=g_hash_table_new(g_str_hash, g_str_equal);
 423         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
 424                 names=g_strdup(country_table[i].names);
 425                 str=names;
 426                 while ((tok=strtok(str, ","))) {
 427                         str=NULL;
 428                         g_hash_table_insert(country_table_hash, tok,  (gpointer)&country_table[i]);
 429                 }
 430         }
 431 }
 432 #endif
 433
 434
 435
 436 static int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
 437 static int in_way, in_node, in_relation;
 438
 439 static void
 440 sig_alrm(int sig)
 441 {
 442 #ifndef _WIN32
 443         signal(SIGALRM, sig_alrm);
 444 #endif
 445         alarm(30);
 446         fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
 447 }
 448
 449 struct item_bin {
 450         int len;
 451         enum item_type type;
 452         int clen;
 453 } item;
 454
 455 struct attr_bin {
 456         int len;
 457         enum attr_type type;
 458 };
 459
 460 struct attr_bin label_attr = {
 461         0, attr_label
 462 };
 463 char label_attr_buffer[BUFFER_SIZE];
 464
 465 struct attr_bin town_name_attr = {
 466         0, attr_town_name
 467 };
 468
 469 struct attr_bin street_name_attr = {
 470         0, attr_street_name
 471 };
 472
 473 struct attr_bin street_name_systematic_attr = {
 474         0, attr_street_name_systematic
 475 };
 476 char street_name_systematic_attr_buffer[BUFFER_SIZE];
 477
 478 struct attr_bin debug_attr = {
 479         0, attr_debug
 480 };
 481 char debug_attr_buffer[BUFFER_SIZE];
 482
 483 struct attr_bin flags_attr = {
 484         0, attr_flags
 485 };
 486 int flags_attr_value;
 487
 488 char is_in_buffer[BUFFER_SIZE];
 489
 490
 491 static void write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level);
 492
 493 static void
 494 pad_text_attr(struct attr_bin *a, char *buffer)
 495 {
 496         int l;
 497         if (buffer && buffer[0]) {
 498                 l=strlen(buffer)+1;
 499                 while (l % 4)
 500                         buffer[l++]='\0';
 501                 a->len=l/4+1;
 502         } else
 503                 a->len=0;
 504 }
 505
 506 static int
 507 xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
 508 {
 509         int len=strlen(attribute);
 510         char *pos,*i,s,attr[len+2];
 511         strcpy(attr, attribute);
 512         strcpy(attr+len, "=");
 513         pos=strstr(xml, attr);
 514         if (! pos)
 515                 return 0;
 516         pos+=len+1;
 517         s=*pos++;
 518         if (! s)
 519                 return 0;
 520         i=strchr(pos, s);
 521         if (! i)
 522                 return 0;
 523         if (i - pos > buffer_size) {
 524                 fprintf(stderr,"Buffer overflow %d vs %d\n", i-pos, buffer_size);
 525                 return 0;
 526         }
 527         strncpy(buffer, pos, i-pos);
 528         buffer[i-pos]='\0';
 529         return 1;
 530 }
 531
 532 static int node_is_tagged;
 533
 534 static void
 535 add_tag(char *k, char *v)
 536 {
 537         GHashTable *value_hash;
 538         enum item_type type;
 539         int idx,level=2;
 540         char buffer[BUFFER_SIZE*2+2];
 541         if (! strcmp(k,"ele"))
 542                 level=9;
 543         if (! strcmp(k,"time"))
 544                 level=9;
 545         if (! strcmp(k,"created_by"))
 546                 level=9;
 547         if (! strncmp(k,"tiger:",6) || !strcmp(k,"AND_nodes"))
 548                 level=9;
 549         if (! strcmp(k,"converted_by") || ! strcmp(k,"source"))
 550                 level=8;
 551         if (! strncmp(k,"osmarender:",11) || !strncmp(k,"svg:",4))
 552                 level=8;
 553         if (! strcmp(k,"layer"))
 554                 level=7;
 555         if (! strcasecmp(v,"true") || ! strcasecmp(v,"yes"))
 556                 v="1";
 557         if (! strcmp(k,"oneway")) {
 558                 if (!strcmp(v,"1")) {
 559                         flags_attr_value=AF_ONEWAY;
 560                         flags_attr.len=2;
 561                 }
 562                 if (! strcmp(v,"-1")) {
 563                         flags_attr_value=AF_ONEWAYREV;
 564                         flags_attr.len=2;
 565                 }
 566                 if (!in_way)
 567                         level=6;
 568                 else
 569                         level=5;
 570         }
 571         if (! strcmp(k,"junction")) {
 572                 if (! strcmp(v,"roundabout")) {
 573                         flags_attr_value=AF_ONEWAY;
 574                         flags_attr.len=2;
 575                 }
 576         }
 577         if (! strcmp(k,"maxspeed")) {
 578                 level=5;
 579         }
 580         if (! strcmp(k,"bicycle")) {
 581                 level=5;
 582         }
 583         if (! strcmp(k,"foot")) {
 584                 level=5;
 585         }
 586         if (! strcmp(k,"note"))
 587                 level=5;
 588         if (! strcmp(k,"name")) {
 589                 strcpy(label_attr_buffer, v);
 590                 pad_text_attr(&label_attr, label_attr_buffer);
 591                 level=5;
 592         }
 593         if (! strcmp(k,"ref")) {
 594                 if (in_way) {
 595                         strcpy(street_name_systematic_attr_buffer, v);
 596                         pad_text_attr(&street_name_systematic_attr, street_name_systematic_attr_buffer);
 597                 }
 598                 level=5;
 599         }
 600         if (! strcmp(k,"is_in")) {
 601                 strcpy(is_in_buffer, v);
 602                 level=5;
 603         }
 604         if (! strcmp(k,"lanes")) {
 605                 level=5;
 606         }
 607         if (attr_debug_level >= level) {
 608                 int bytes_left = sizeof( debug_attr_buffer ) - strlen(debug_attr_buffer) - 1;
 609                 if ( bytes_left > 0 )
 610                 {
 611                         snprintf(debug_attr_buffer+strlen(debug_attr_buffer), bytes_left,  " %s=%s", k, v);
 612                         debug_attr_buffer[ sizeof( debug_attr_buffer ) -  1 ] = '\0';
 613                         node_is_tagged=1;
 614                 }
 615         }
 616         if (level < 6)
 617                 node_is_tagged=1;
 618         if (level >= 5)
 619                 return;
 620
 621         strcpy(buffer,"*=*");
 622         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 623                 attr_present[idx]=1;
 624
 625         sprintf(buffer,"%s=*", k);
 626         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 627                 attr_present[idx]=2;
 628
 629         sprintf(buffer,"*=%s", v);
 630         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 631                 attr_present[idx]=2;
 632
 633         sprintf(buffer,"%s=%s", k, v);
 634         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 635                 attr_present[idx]=4;
 636 }
 637
 638 struct entity {
 639         char *entity;
 640         char c;
 641 } entities[]= {
 642         {"&quot;",'"'},
 643         {"&apos;",'\''},
 644         {"&amp;",'&'},
 645         {"&lt;",'<'},
 646         {"&gt;",'>'},
 647 };
 648
 649 static void
 650 decode_entities(char *buffer)
 651 {
 652         char *pos=buffer;
 653         int i,len,found;
 654
 655         while ((pos=strchr(pos, '&'))) {
 656                 found=0;
 657                 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) {
 658                         len=strlen(entities[i].entity);
 659                         if (!strncmp(pos, entities[i].entity, len)) {
 660                                 *pos=entities[i].c;
 661                                 memmove(pos+1, pos+len, strlen(pos+len)+1);
 662                                 found=1;
 663                                 break;
 664                         }
 665                 }
 666                 pos++;
 667         }
 668 }
 669
 670 static int
 671 parse_tag(char *p)
 672 {
 673         char k_buffer[BUFFER_SIZE];
 674         char v_buffer[BUFFER_SIZE];
 675         if (!xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
 676                 return 0;
 677         if (!xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
 678                 return 0;
 679         decode_entities(v_buffer);
 680         add_tag(k_buffer, v_buffer);
 681         return 1;
 682 }
 683
 684
 685 struct buffer {
 686         int malloced_step;
 687         size_t malloced;
 688         unsigned char *base;
 689         size_t size;
 690 };
 691
 692 static struct tile_head {
 693         int num_subtiles;
 694         int total_size;
 695         char *name;
 696         char *zip_data;
 697         int total_size_used;
 698         int zipnum;
 699         int process;
 700         struct tile_head *next;
 701         // char subtiles[0];
 702 } *tile_head_root;
 703
 704
 705 int coord_count;
 706
 707 struct node_item {
 708         int id;
 709         char ref_node;
 710         char ref_way;
 711         char ref_ref;
 712         char dummy;
 713         struct coord c;
 714 };
 715
 716 static struct buffer node_buffer = {
 717         64*1024*1024,
 718 };
 719
 720
 721 static char** th_get_subtile( const struct tile_head* th, int idx )
 722 {
 723         char* subtile_ptr = NULL;
 724         subtile_ptr = (char*)th + sizeof( struct tile_head ) + idx * sizeof( char *);
 725         return (char**)subtile_ptr;
 726 }
 727
 728 static void
 729 extend_buffer(struct buffer *b)
 730 {
 731         b->malloced+=b->malloced_step;
 732         b->base=realloc(b->base, b->malloced);
 733         if (b->base == NULL) {
 734                 fprintf(stderr,"realloc of %d bytes failed\n",b->malloced);
 735                 exit(1);
 736         }
 737
 738 }
 739
 740 int nodeid_last;
 741 GHashTable *node_hash;
 742
 743 static void
 744 node_buffer_to_hash(void)
 745 {
 746         int i,count=node_buffer.size/sizeof(struct node_item);
 747         struct node_item *ni=(struct node_item *)node_buffer.base;
 748         for (i = 0 ; i < count ; i++)
 749                 g_hash_table_insert(node_hash, (gpointer)(ni[i].id), (gpointer)i);
 750 }
 751
 752 static struct node_item *ni;
 753
 754 static void
 755 add_node(int id, double lat, double lon)
 756 {
 757         if (node_buffer.size + sizeof(struct node_item) > node_buffer.malloced)
 758                 extend_buffer(&node_buffer);
 759         node_is_tagged=0;
 760         nodeid=id;
 761         item.type=type_point_unkn;
 762         label_attr.len=0;
 763         town_name_attr.len=0;
 764         debug_attr.len=0;
 765         is_in_buffer[0]='\0';
 766         sprintf(debug_attr_buffer,"nodeid=%d", nodeid);
 767         ni=(struct node_item *)(node_buffer.base+node_buffer.size);
 768         ni->id=id;
 769         ni->ref_node=0;
 770         ni->ref_way=0;
 771         ni->ref_ref=0;
 772         ni->dummy=0;
 773         ni->c.x=lon*6371000.0*M_PI/180;
 774         ni->c.y=log(tan(M_PI_4+lat*M_PI/360))*6371000.0;
 775         node_buffer.size+=sizeof(struct node_item);
 776         if (! node_hash) {
 777                 if (ni->id > nodeid_last) {
 778                         nodeid_last=ni->id;
 779                 } else {
 780                         fprintf(stderr,"INFO: Nodes out of sequence (new %d vs old %d), adding hash\n", ni->id, nodeid_last);
 781                         node_hash=g_hash_table_new(NULL, NULL);
 782                         node_buffer_to_hash();
 783                 }
 784         } else
 785                 if (!g_hash_table_lookup(node_hash, (gpointer)(ni->id)))
 786                         g_hash_table_insert(node_hash, (gpointer)(ni->id), (gpointer)(ni-(struct node_item *)node_buffer.base));
 787                 else {
 788                         node_buffer.size-=sizeof(struct node_item);
 789                         nodeid=0;
 790                 }
 791
 792 }
 793
 794 static int
 795 parse_node(char *p)
 796 {
 797         char id_buffer[BUFFER_SIZE];
 798         char lat_buffer[BUFFER_SIZE];
 799         char lon_buffer[BUFFER_SIZE];
 800         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 801                 return 0;
 802         if (!xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
 803                 return 0;
 804         if (!xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
 805                 return 0;
 806         add_node(atoi(id_buffer), atof(lat_buffer), atof(lon_buffer));
 807         return 1;
 808 }
 809
 810
 811 static struct node_item *
 812 node_item_get(int id)
 813 {
 814         struct node_item *ni=(struct node_item *)(node_buffer.base);
 815         int count=node_buffer.size/sizeof(struct node_item);
 816         int interval=count/4;
 817         int p=count/2;
 818         if (node_hash) {
 819                 int i;
 820                 i=(int)(g_hash_table_lookup(node_hash, (gpointer)id));
 821                 return ni+i;
 822         }
 823         while (ni[p].id != id) {
 824 #if 0
 825                 fprintf(stderr,"p=%d count=%d interval=%d id=%d ni[p].id=%d\n", p, count, interval, id, ni[p].id);
 826 #endif
 827                 if (ni[p].id < id) {
 828                         p+=interval;
 829                         if (interval == 1) {
 830                                 if (p >= count)
 831                                         return NULL;
 832                                 if (ni[p].id > id)
 833                                         return NULL;
 834                         } else {
 835                                 if (p >= count)
 836                                         p=count-1;
 837                         }
 838                 } else {
 839                         p-=interval;
 840                         if (interval == 1) {
 841                                 if (p < 0)
 842                                         return NULL;
 843                                 if (ni[p].id < id)
 844                                         return NULL;
 845                         } else {
 846                                 if (p < 0)
 847                                         p=0;
 848                         }
 849                 }
 850                 if (interval > 1)
 851                         interval/=2;
 852         }
 853
 854         return &ni[p];
 855 }
 856
 857 static void
 858 node_ref_way(int id)
 859 {
 860         struct node_item *ni=node_item_get(id);
 861         if (! ni) {
 862                 fprintf(stderr,"WARNING: node id %d not found\n", id);
 863                 return;
 864         }
 865         ni->ref_way++;
 866 }
 867
 868
 869 static void
 870 add_way(int id)
 871 {
 872         wayid=id;
 873         coord_count=0;
 874         item.type=type_street_unkn;
 875         label_attr.len=0;
 876         street_name_attr.len=0;
 877         street_name_systematic_attr.len=0;
 878         debug_attr.len=0;
 879         flags_attr.len=0;
 880         sprintf(debug_attr_buffer,"wayid=%d", wayid);
 881 }
 882
 883 static int
 884 parse_way(char *p)
 885 {
 886         char id_buffer[BUFFER_SIZE];
 887         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 888                 return 0;
 889         add_way(atoi(id_buffer));
 890         return 1;
 891 }
 892
 893 static int
 894 parse_relation(char *p)
 895 {
 896         debug_attr_buffer[0]='\0';
 897         return 1;
 898 }
 899
 900 static void
 901 write_attr(FILE *out, struct attr_bin *attr, void *buffer)
 902 {
 903         if (attr->len) {
 904                 fwrite(attr, sizeof(*attr), 1, out);
 905                 fwrite(buffer, (attr->len-1)*4, 1, out);
 906         }
 907 }
 908
 909 static int
 910 attr_longest_match(struct attr_mapping **mapping, int mapping_count, enum item_type *types, int types_count)
 911 {
 912         int i,j,longest=0,ret=0,sum,val;
 913         struct attr_mapping *curr;
 914         for (i = 0 ; i < mapping_count ; i++) {
 915                 sum=0;
 916                 curr=mapping[i];
 917                 for (j = 0 ; j < curr->attr_present_idx_count ; j++) {
 918                         val=attr_present[curr->attr_present_idx[j]];
 919                         if (val)
 920                                 sum+=val;
 921                         else {
 922                                 sum=-1;
 923                                 break;
 924                         }
 925                 }
 926                 if (sum > longest) {
 927                         longest=sum;
 928                         ret=0;
 929                 }
 930                 if (sum > 0 && sum == longest && ret < types_count)
 931                         types[ret++]=curr->type;
 932         }
 933         memset(attr_present, 0, sizeof(*attr_present)*attr_present_count);
 934         return ret;
 935 }
 936
 937 static void
 938 end_way(FILE *out)
 939 {
 940         int alen=0,count;
 941         enum item_type types[5];
 942
 943         if (! out)
 944                 return;
 945         if (dedupe_ways_hash) {
 946                 if (g_hash_table_lookup(dedupe_ways_hash, (gpointer)wayid))
 947                         return;
 948                 g_hash_table_insert(dedupe_ways_hash, (gpointer)wayid, (gpointer)1);
 949         }
 950         count=attr_longest_match(attr_mapping_way, attr_mapping_way_count, types, sizeof(types)/sizeof(enum item_type));
 951         pad_text_attr(&debug_attr, debug_attr_buffer);
 952         if (label_attr.len)
 953                 alen+=label_attr.len+1;
 954         if (street_name_systematic_attr.len)
 955                 alen+=street_name_systematic_attr.len+1;
 956         if (debug_attr.len)
 957                 alen+=debug_attr.len+1;
 958         if (flags_attr.len)
 959                 alen+=flags_attr.len+1;
 960         if (count)
 961                 item.type=types[0];
 962         else
 963                 item.type=type_street_unkn;
 964         if (coverage && IS_STREET(item))
 965                 item.type=type_coverage;
 966         item.clen=coord_count*2;
 967         item.len=item.clen+2+alen;
 968         fwrite(&item, sizeof(item), 1, out);
 969         fwrite(coord_buffer, coord_count*sizeof(struct coord), 1, out);
 970         if (IS_STREET(item)) {
 971                 street_name_attr.len=label_attr.len;
 972                 write_attr(out, &street_name_attr, label_attr_buffer);
 973         } else
 974                 write_attr(out, &label_attr, label_attr_buffer);
 975         write_attr(out, &street_name_systematic_attr, street_name_systematic_attr_buffer);
 976         write_attr(out, &debug_attr, debug_attr_buffer);
 977         write_attr(out, &flags_attr, &flags_attr_value);
 978 }
 979
 980 static void
 981 end_node(FILE *out)
 982 {
 983         int alen=0,conflict=0,count;
 984         enum item_type types[5];
 985         struct country_table *result=NULL, *lookup;
 986         if (!out || ! node_is_tagged || ! nodeid)
 987                 return;
 988         count=attr_longest_match(attr_mapping_node, attr_mapping_node_count, types, sizeof(types)/sizeof(enum item_type));
 989         pad_text_attr(&debug_attr, debug_attr_buffer);
 990         if (label_attr.len)
 991                 alen+=label_attr.len+1;
 992         if (debug_attr.len)
 993                 alen+=debug_attr.len+1;
 994         if (count)
 995                 item.type=types[0];
 996         else
 997                 item.type=type_point_unkn;
 998         item.clen=2;
 999         item.len=item.clen+2+alen;
1000         fwrite(&item, sizeof(item), 1, out);
1001         fwrite(&ni->c, 1*sizeof(struct coord), 1, out);
1002         if (IS_TOWN(item)) {
1003                 town_name_attr.len=label_attr.len;
1004                 write_attr(out, &town_name_attr, label_attr_buffer);
1005         } else
1006                 write_attr(out, &label_attr, label_attr_buffer);
1007         write_attr(out, &debug_attr, debug_attr_buffer);
1008 #ifdef GENERATE_INDEX
1009         if (IS_TOWN(item) && town_name_attr.len) {
1010                 char *tok,*buf=is_in_buffer;
1011                 while ((tok=strtok(buf, ","))) {
1012                         while (*tok==' ')
1013                                 tok++;
1014                         lookup=g_hash_table_lookup(country_table_hash,tok);
1015                         if (lookup) {
1016                                 if (result && result->countryid != lookup->countryid) {
1017                                         fprintf(stderr,"conflict for %s %s country %d vs %d\n", label_attr_buffer, debug_attr_buffer, lookup->countryid, result->countryid);
1018                                         conflict=1;
1019                                 } else
1020                                         result=lookup;
1021                         }
1022                         buf=NULL;
1023                 }
1024                 if (result && !conflict) {
1025                         if (!result->file) {
1026                                 char *name=g_strdup_printf("country_%d.bin.unsorted", result->countryid);
1027                                 result->file=fopen(name,"w");
1028                                 g_free(name);
1029                         }
1030                         if (result->file) {
1031                                 item.clen=2;
1032                                 item.len=item.clen+2+label_attr.len+1;
1033                                 fwrite(&item, sizeof(item), 1, result->file);
1034                                 fwrite(&ni->c, 1*sizeof(struct coord), 1, result->file);
1035                                 write_attr(result->file, &town_name_attr, label_attr_buffer);
1036                                 result->count++;
1037                                 result->size+=(item.clen+3+label_attr.len+1)*4;
1038                         }
1039
1040                 }
1041         }
1042 #endif
1043         processed_nodes_out++;
1044 }
1045
1046 static int
1047 sort_countries_compare(const void *p1, const void *p2)
1048 {
1049         struct item_bin *ib1=*((struct item_bin **)p1),*ib2=*((struct item_bin **)p2);
1050         struct attr_bin *attr1,*attr2;
1051         char *s1,*s2;
1052         assert(ib1->clen==2);
1053         assert(ib2->clen==2);
1054         attr1=(struct attr_bin *)((int *)(ib1+1)+ib1->clen);
1055         attr2=(struct attr_bin *)((int *)(ib2+1)+ib1->clen);
1056         assert(attr1->type == attr_town_name);
1057         assert(attr2->type == attr_town_name);
1058         s1=(char *)(attr1+1);
1059         s2=(char *)(attr2+1);
1060         return strcmp(s1, s2);
1061 #if 0
1062         fprintf(stderr,"sort_countries_compare p1=%p p2=%p %s %s\n",p1,p2,s1,s2);
1063 #endif
1064         return 0;
1065 }
1066
1067 #ifdef GENERATE_INDEX
1068 static void
1069 sort_countries(void)
1070 {
1071         int i,j;
1072         struct country_table *co;
1073         struct coord *c;
1074         struct item_bin *ib;
1075         FILE *f;
1076         char *p,*buffer,**idx,*name;
1077         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1078                 co=&country_table[i];
1079                 if (co->file) {
1080                         fclose(co->file);
1081                         co->file=NULL;
1082                 }
1083                 if (co->size) {
1084                         buffer=malloc(co->size);
1085                         assert(buffer != NULL);
1086                         idx=malloc(co->count*sizeof(void *));
1087                         assert(idx != NULL);
1088                         name=g_strdup_printf("country_%d.bin.unsorted", co->countryid);
1089                         f=fopen(name,"r");
1090                         assert(f != NULL);
1091                         fread(buffer, co->size, 1, f);
1092                         fclose(f);
1093                         unlink(name);
1094                         g_free(name);
1095                         p=buffer;
1096                         for (j = 0 ; j < co->count ; j++) {
1097                                 idx[j]=p;
1098                                 p+=(*((int *)p)+1)*4;
1099                         }
1100                         qsort(idx, co->count, sizeof(void *), sort_countries_compare);
1101                         name=g_strdup_printf("country_%d.bin", co->countryid);
1102                         f=fopen(name,"w");
1103                         for (j = 0 ; j < co->count ; j++) {
1104                                 ib=(struct item_bin *)(idx[j]);
1105                                 c=(struct coord *)(ib+1);
1106                                 fwrite(ib, (ib->len+1)*4, 1, f);
1107                                 if (j)
1108                                         bbox_extend(c, &co->r);
1109                                 else
1110                                         co->r.l=*c;
1111                                         co->r.h=*c;
1112                         }
1113                         fclose(f);
1114                 }
1115         }
1116 }
1117 #endif
1118
1119 static void
1120 add_nd(char *p, int ref)
1121 {
1122         int len;
1123         struct node_item *ni;
1124         ni=node_item_get(ref);
1125         if (ni) {
1126 #if 0
1127                 coord_buffer[coord_count++]=ni->c;
1128 #else
1129                 SET_REF(coord_buffer[coord_count], ref);
1130                 coord_count++;
1131 #endif
1132                 ni->ref_way++;
1133         } else {
1134                 len=strlen(p);
1135                 if (len > 0 && p[len-1]=='\n')
1136                         p[len-1]='\0';
1137                 fprintf(stderr,"WARNING: way %d: node %d not found (%s)\n",wayid,ref,p);
1138         }
1139         if (coord_count > 65536) {
1140                 fprintf(stderr,"ERROR: Overflow\n");
1141                 exit(1);
1142         }
1143 }
1144
1145 static int
1146 parse_nd(char *p)
1147 {
1148         char ref_buffer[BUFFER_SIZE];
1149         if (!xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
1150                 return 0;
1151         add_nd(p, atoi(ref_buffer));
1152         return 1;
1153 }
1154
1155
1156 static void
1157 save_buffer(char *filename, struct buffer *b)
1158 {
1159         FILE *f;
1160         f=fopen(filename,"wb+");
1161         fwrite(b->base, b->size, 1, f);
1162         fclose(f);
1163 }
1164
1165 static void
1166 load_buffer(char *filename, struct buffer *b)
1167 {
1168         FILE *f;
1169         if (b->base)
1170                 free(b->base);
1171         b->malloced=0;
1172         f=fopen(filename,"rb");
1173         fseek(f, 0, SEEK_END);
1174         b->size=b->malloced=ftell(f);
1175         fprintf(stderr,"reading %d bytes from %s\n", b->size, filename);
1176         fseek(f, 0, SEEK_SET);
1177         b->base=malloc(b->size);
1178         assert(b->base != NULL);
1179         fread(b->base, b->size, 1, f);
1180         fclose(f);
1181 }
1182
1183 static int
1184 phase1(FILE *in, FILE *out_ways, FILE *out_nodes)
1185 {
1186         int size=4096;
1187         char buffer[size];
1188         char *p;
1189         sig_alrm(0);
1190         while (fgets(buffer, size, in)) {
1191                 p=strchr(buffer,'<');
1192                 if (! p) {
1193                         fprintf(stderr,"WARNING: wrong line %s\n", buffer);
1194                         continue;
1195                 }
1196                 if (!strncmp(p, "<?xml ",6)) {
1197                 } else if (!strncmp(p, "<osm ",5)) {
1198                 } else if (!strncmp(p, "<bound ",7)) {
1199                 } else if (!strncmp(p, "<node ",6)) {
1200                         if (!parse_node(p))
1201                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1202                         in_node=1;
1203                         processed_nodes++;
1204                 } else if (!strncmp(p, "<tag ",5)) {
1205                         if (!parse_tag(p))
1206                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1207                 } else if (!strncmp(p, "<way ",5)) {
1208                         in_way=1;
1209                         if (!parse_way(p))
1210                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1211                         processed_ways++;
1212                 } else if (!strncmp(p, "<nd ",4)) {
1213                         if (!parse_nd(p))
1214                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1215                 } else if (!strncmp(p, "<relation ",10)) {
1216                         in_relation=1;
1217                         if (!parse_relation(p))
1218                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1219                         processed_relations++;
1220                 } else if (!strncmp(p, "<member ",8)) {
1221                 } else if (!strncmp(p, "</node>",7)) {
1222                         in_node=0;
1223                         end_node(out_nodes);
1224                 } else if (!strncmp(p, "</way>",6)) {
1225                         in_way=0;
1226                         end_way(out_ways);
1227                 } else if (!strncmp(p, "</relation>",11)) {
1228                         in_relation=0;
1229                 } else if (!strncmp(p, "</osm>",6)) {
1230                 } else {
1231                         fprintf(stderr,"WARNING: unknown tag in %s\n", buffer);
1232                 }
1233         }
1234         sig_alrm(0);
1235         alarm(0);
1236         return 1;
1237 }
1238
1239 static char buffer[150000];
1240
1241 int bytes_read=0;
1242
1243 static struct item_bin *
1244 read_item(FILE *in)
1245 {
1246         struct item_bin *ib=(struct item_bin *) buffer;
1247         int r,s;
1248         r=fread(ib, sizeof(*ib), 1, in);
1249         if (r != 1)
1250                 return NULL;
1251         bytes_read+=r;
1252         assert((ib->len+1) < sizeof(buffer));
1253         s=(ib->len+1)*4-sizeof(*ib);
1254         r=fread(ib+1, s, 1, in);
1255         if (r != 1)
1256                 return NULL;
1257         bytes_read+=r;
1258         return ib;
1259 }
1260
1261 static void
1262 bbox_extend(struct coord *c, struct rect *r)
1263 {
1264         if (c->x < r->l.x)
1265                 r->l.x=c->x;
1266         if (c->y < r->l.y)
1267                 r->l.y=c->y;
1268         if (c->x > r->h.x)
1269                 r->h.x=c->x;
1270         if (c->y > r->h.y)
1271                 r->h.y=c->y;
1272 }
1273
1274 static void
1275 bbox(struct coord *c, int count, struct rect *r)
1276 {
1277         if (! count)
1278                 return;
1279         r->l=*c;
1280         r->h=*c;
1281         while (--count) {
1282                 c++;
1283                 bbox_extend(c, r);
1284         }
1285 }
1286
1287 static int
1288 contains_bbox(int xl, int yl, int xh, int yh, struct rect *r)
1289 {
1290         if (r->h.x < xl || r->h.x > xh) {
1291                 return 0;
1292         }
1293         if (r->l.x > xh || r->l.x < xl) {
1294                 return 0;
1295         }
1296         if (r->h.y < yl || r->h.y > yh) {
1297                 return 0;
1298         }
1299         if (r->l.y > yh || r->l.y < yl) {
1300                 return 0;
1301         }
1302         return 1;
1303
1304 }
1305 struct rect world_bbox = {
1306         { -20000000, -20000000},
1307         {  20000000,  20000000},
1308 };
1309
1310 static void
1311 tile(struct rect *r, char *ret, int max)
1312 {
1313         int x0,x1,x2,x3,x4;
1314         int y0,y1,y2,y3,y4;
1315         int i;
1316         x0=world_bbox.l.x;
1317         y0=world_bbox.l.y;
1318         x4=world_bbox.h.x;
1319         y4=world_bbox.h.y;
1320         for (i = 0 ; i < max ; i++) {
1321                 x2=(x0+x4)/2;
1322                 y2=(y0+y4)/2;
1323                 x1=(x0+x2)/2;
1324                 y1=(y0+y2)/2;
1325                 x3=(x2+x4)/2;
1326                 y3=(y2+y4)/2;
1327                 if (     contains_bbox(x0,y0,x2,y2,r)) {
1328                         strcat(ret,"d");
1329                         x4=x2;
1330                         y4=y2;
1331                 } else if (contains_bbox(x2,y0,x4,y2,r)) {
1332                         strcat(ret,"c");
1333                         x0=x2;
1334                         y4=y2;
1335                 } else if (contains_bbox(x0,y2,x2,y4,r)) {
1336                         strcat(ret,"b");
1337                         x4=x2;
1338                         y0=y2;
1339                 } else if (contains_bbox(x2,y2,x4,y4,r)) {
1340                         strcat(ret,"a");
1341                         x0=x2;
1342                         y0=y2;
1343                 } else
1344                         return;
1345         }
1346 }
1347
1348 static void
1349 tile_bbox(char *tile, struct rect *r)
1350 {
1351         struct coord c;
1352         *r=world_bbox;
1353         while (*tile) {
1354                 c.x=(r->l.x+r->h.x)/2;
1355                 c.y=(r->l.y+r->h.y)/2;
1356                 switch (*tile) {
1357                 case 'a':
1358                         r->l.x=c.x;
1359                         r->l.y=c.y;
1360                         break;
1361                 case 'b':
1362                         r->h.x=c.x;
1363                         r->l.y=c.y;
1364                         break;
1365                 case 'c':
1366                         r->l.x=c.x;
1367                         r->h.y=c.y;
1368                         break;
1369                 case 'd':
1370                         r->h.x=c.x;
1371                         r->h.y=c.y;
1372                         break;
1373                 }
1374                 tile++;
1375         }
1376 }
1377
1378 GHashTable *tile_hash;
1379 GHashTable *tile_hash2;
1380
1381 static void
1382 tile_extend(char *tile, struct item_bin *ib, GList **tiles_list)
1383 {
1384         struct tile_head *th=NULL;
1385         if (debug_tile(tile))
1386                 fprintf(stderr,"Tile:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1387         if (tile_hash2)
1388                 th=g_hash_table_lookup(tile_hash2, tile);
1389         if (!th)
1390                 th=g_hash_table_lookup(tile_hash, tile);
1391         if (! th) {
1392                 th=malloc(sizeof(struct tile_head)+ sizeof( char* ) );
1393                 assert(th != NULL);
1394                 // strcpy(th->subtiles, tile);
1395                 th->num_subtiles=1;
1396                 th->total_size=0;
1397                 th->total_size_used=0;
1398                 th->zipnum=0;
1399                 th->zip_data=NULL;
1400                 th->name=string_hash_lookup(tile);
1401                 *th_get_subtile( th, 0 ) = th->name;
1402
1403                 if (tile_hash2)
1404                         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1405                 if (tiles_list)
1406                         *tiles_list=g_list_append(*tiles_list, string_hash_lookup( th->name ) );
1407                 processed_tiles++;
1408                 if (debug_tile(tile))
1409                         fprintf(stderr,"new '%s'\n", tile);
1410         }
1411         th->total_size+=ib->len*4+4;
1412         if (debug_tile(tile))
1413                 fprintf(stderr,"New total size of %s(%p):%d\n", th->name, th, th->total_size);
1414         g_hash_table_insert(tile_hash, string_hash_lookup( th->name ), th);
1415 }
1416
1417 static int
1418 tile_data_size(char *tile)
1419 {
1420         struct tile_head *th;
1421         th=g_hash_table_lookup(tile_hash, tile);
1422         if (! th)
1423                 return 0;
1424         return th->total_size;
1425 }
1426
1427 static int
1428 merge_tile(char *base, char *sub)
1429 {
1430         struct tile_head *thb, *ths;
1431         thb=g_hash_table_lookup(tile_hash, base);
1432         ths=g_hash_table_lookup(tile_hash, sub);
1433         if (! ths)
1434                 return 0;
1435         if (debug_tile(base) || debug_tile(sub))
1436                 fprintf(stderr,"merging '%s'(%p) (%d) with '%s'(%p) (%d)\n", base, thb, thb ? thb->total_size : 0, sub, ths, ths->total_size);
1437         if (! thb) {
1438                 thb=ths;
1439                 g_hash_table_remove(tile_hash, sub);
1440                 thb->name=string_hash_lookup(base);
1441                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1442
1443         } else {
1444                 thb=realloc(thb, sizeof(struct tile_head)+( ths->num_subtiles+thb->num_subtiles ) * sizeof( char*) );
1445                 assert(thb != NULL);
1446                 memcpy( th_get_subtile( thb, thb->num_subtiles ), th_get_subtile( ths, 0 ), ths->num_subtiles * sizeof( char*) );
1447                 thb->num_subtiles+=ths->num_subtiles;
1448                 thb->total_size+=ths->total_size;
1449                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1450                 g_hash_table_remove(tile_hash, sub);
1451                 g_free(ths);
1452         }
1453         return 1;
1454 }
1455
1456
1457 static void
1458 get_tiles_list_func(char *key, struct tile_head *th, GList **list)
1459 {
1460         *list=g_list_prepend(*list, key);
1461 }
1462
1463 static GList *
1464 get_tiles_list(void)
1465 {
1466         GList *ret=NULL;
1467         g_hash_table_foreach(tile_hash, (GHFunc)get_tiles_list_func, &ret);
1468         return ret;
1469 }
1470
1471 #if 0
1472 static void
1473 write_tile(char *key, struct tile_head *th, gpointer dummy)
1474 {
1475         FILE *f;
1476         char buffer[1024];
1477         fprintf(stderr,"DEBUG: Writing %s\n", key);
1478         strcpy(buffer,"tiles/");
1479         strcat(buffer,key);
1480 #if 0
1481         strcat(buffer,".bin");
1482 #endif
1483         f=fopen(buffer, "wb+");
1484         while (th) {
1485                 fwrite(th->data, th->size, 1, f);
1486                 th=th->next;
1487         }
1488         fclose(f);
1489 }
1490 #endif
1491
1492 static void
1493 write_item(char *tile, struct item_bin *ib)
1494 {
1495         struct tile_head *th;
1496         int size;
1497
1498         th=g_hash_table_lookup(tile_hash2, tile);
1499         if (! th)
1500                 th=g_hash_table_lookup(tile_hash, tile);
1501         if (th) {
1502                 if (th->process != 0 && th->process != 1) {
1503                         fprintf(stderr,"error with tile '%s' of length %d\n", tile, strlen(tile));
1504                         abort();
1505                 }
1506                 if (! th->process)
1507                         return;
1508                 if (debug_tile(tile))
1509                         fprintf(stderr,"Data:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1510                 size=(ib->len+1)*4;
1511                 if (th->total_size_used+size > th->total_size) {
1512                         fprintf(stderr,"Overflow in tile %s (used %d max %d item %d)\n", tile, th->total_size_used, th->total_size, size);
1513                         exit(1);
1514                         return;
1515                 }
1516                 memcpy(th->zip_data+th->total_size_used, ib, size);
1517                 th->total_size_used+=size;
1518         } else {
1519                 fprintf(stderr,"no tile hash found for %s\n", tile);
1520                 exit(1);
1521         }
1522 }
1523
1524
1525 static void
1526 write_item_part(FILE *out, struct item_bin *orig, int first, int last)
1527 {
1528         struct item_bin new;
1529         struct coord *c=(struct coord *)(orig+1);
1530         char *attr=(char *)(c+orig->clen/2);
1531         int attr_len=orig->len-orig->clen-2;
1532         processed_ways++;
1533         new.type=orig->type;
1534         new.clen=(last-first+1)*2;
1535         new.len=new.clen+attr_len+2;
1536 #if 0
1537         fprintf(stderr,"first %d last %d type 0x%x len %d clen %d attr_len %d\n", first, last, new.type, new.len, new.clen, attr_len);
1538 #endif
1539         fwrite(&new, sizeof(new), 1, out);
1540         fwrite(c+first, new.clen*4, 1, out);
1541         fwrite(attr, attr_len*4, 1, out);
1542 }
1543
1544 static int
1545 phase2(FILE *in, FILE *out)
1546 {
1547         struct coord *c;
1548         int i,ccount,last,ndref;
1549         struct item_bin *ib;
1550         struct node_item *ni;
1551
1552         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1553         sig_alrm(0);
1554         while ((ib=read_item(in))) {
1555 #if 0
1556                 fprintf(stderr,"type 0x%x len %d clen %d\n", ib->type, ib->len, ib->clen);
1557 #endif
1558                 ccount=ib->clen/2;
1559                 c=(struct coord *)(ib+1);
1560                 last=0;
1561                 for (i = 0 ; i < ccount ; i++) {
1562                         if (IS_REF(c[i])) {
1563                                 ndref=REF(c[i]);
1564                                 ni=node_item_get(ndref);
1565 #if 0
1566                                 fprintf(stderr,"ni=%p\n", ni);
1567 #endif
1568                                 c[i]=ni->c;
1569                                 if (ni->ref_way > 1 && i != 0 && i != ccount-1 && IS_STREET(*ib)) {
1570                                         write_item_part(out, ib, last, i);
1571                                         last=i;
1572                                 }
1573                         }
1574                 }
1575                 write_item_part(out, ib, last, ccount-1);
1576         }
1577         sig_alrm(0);
1578         alarm(0);
1579         return 0;
1580 }
1581
1582 static void
1583 phase34_process_file(int phase, FILE *in)
1584 {
1585         struct item_bin *ib;
1586         struct rect r;
1587         char buffer[1024];
1588         int max;
1589
1590         while ((ib=read_item(in))) {
1591                 if (ib->type < 0x80000000)
1592                         processed_nodes++;
1593                 else
1594                         processed_ways++;
1595                 bbox((struct coord *)(ib+1), ib->clen/2, &r);
1596                 buffer[0]='\0';
1597                 max=14;
1598                 if (ib->type == type_street_n_lanes || ib->type == type_highway_city || ib->type == type_highway_land || ib->type == type_ramp)
1599                         max=8;
1600                 if (ib->type == type_street_3_city || ib->type == type_street_4_city || ib->type == type_street_3_land || ib->type == type_street_4_land)
1601                         max=12;
1602
1603                 tile(&r, buffer, max);
1604 #if 0
1605                 fprintf(stderr,"%s\n", buffer);
1606 #endif
1607                 if (phase == 3)
1608                         tile_extend(buffer, ib, NULL);
1609                 else
1610                         write_item(buffer, ib);
1611         }
1612 }
1613
1614 struct index_item {
1615         struct item_bin item;
1616         struct rect r;
1617         struct attr_bin attr_order_limit;
1618         short min;
1619         short max;
1620         struct attr_bin attr_zipfile_ref;
1621         int zipfile_ref;
1622 };
1623
1624 static void
1625 index_submap_add(int phase, struct tile_head *th, GList **tiles_list)
1626 {
1627         struct index_item ii;
1628         int len=strlen(th->name);
1629         char index_tile[len+1];
1630
1631         ii.min=(len > 4) ? len-4 : 0;
1632         ii.max=255;
1633         strcpy(index_tile, th->name);
1634         if (len > 6)
1635                 len=6;
1636         else
1637                 len=0;
1638         index_tile[len]=0;
1639         tile_bbox(th->name, &ii.r);
1640
1641         ii.item.len=sizeof(ii)/4-1;
1642         ii.item.type=type_submap;
1643         ii.item.clen=4;
1644
1645         ii.attr_order_limit.len=2;
1646         ii.attr_order_limit.type=attr_order_limit;
1647
1648         ii.attr_zipfile_ref.len=2;
1649         ii.attr_zipfile_ref.type=attr_zipfile_ref;
1650         ii.zipfile_ref=th->zipnum;
1651
1652         if (phase == 3)
1653                 tile_extend(index_tile, (struct item_bin *)&ii, tiles_list);
1654         else
1655                 write_item(index_tile, (struct item_bin *)&ii);
1656 #if 0
1657         unsigned int *c=(unsigned int *)&ii;
1658         int i;
1659         for (i = 0 ; i < sizeof(ii)/4 ; i++) {
1660                 fprintf(stderr,"%08x ", c[i]);
1661         }
1662         fprintf(stderr,"\n");
1663 #endif
1664 }
1665
1666 static int
1667 add_tile_hash(struct tile_head *th)
1668 {
1669         int idx,len,maxnamelen=0;
1670         char **data;
1671
1672 #if 0
1673         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1674 #endif
1675         for( idx = 0; idx < th->num_subtiles; idx++ ) {
1676
1677         data = th_get_subtile( th, idx );
1678
1679                 if (debug_tile(data) || debug_tile(th->name)) {
1680                         fprintf(stderr,"Parent for '%s' is '%s'\n", *data, th->name);
1681                 }
1682
1683                 g_hash_table_insert(tile_hash2, *data, th);
1684
1685                 len = strlen( *data );
1686
1687                 if (len > maxnamelen) {
1688                         maxnamelen=len;
1689                 }
1690         }
1691         return maxnamelen;
1692 }
1693
1694
1695 static int
1696 create_tile_hash(void)
1697 {
1698         struct tile_head *th;
1699         int len,maxnamelen=0;
1700
1701         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1702         th=tile_head_root;
1703         while (th) {
1704                 len=add_tile_hash(th);
1705                 if (len > maxnamelen)
1706                         maxnamelen=len;
1707                 th=th->next;
1708         }
1709         return maxnamelen;
1710 }
1711
1712 static void
1713 create_tile_hash_list(GList *list)
1714 {
1715         GList *next;
1716         struct tile_head *th;
1717
1718         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1719
1720         fprintf(stderr,"list=%p\n", list);
1721         next=g_list_first(list);
1722         while (next) {
1723                 th=g_hash_table_lookup(tile_hash, next->data);
1724                 if (!th) {
1725                         fprintf(stderr,"No tile found for '%s'\n", (char *)(next->data));
1726                 }
1727                 add_tile_hash(th);
1728                 next=g_list_next(next);
1729         }
1730 }
1731
1732 static void
1733 destroy_tile_hash(void)
1734 {
1735         g_hash_table_destroy(tile_hash2);
1736         tile_hash2=NULL;
1737 }
1738
1739 static int zipnum;
1740
1741 static void write_countrydir(int phase, int maxnamelen);
1742
1743 static void
1744 write_tilesdir(int phase, int maxlen, FILE *out)
1745 {
1746         int idx,len;
1747         GList *tiles_list,*next;
1748         char **data;
1749         struct tile_head *th,**last=NULL;
1750         zipnum=0;
1751
1752         tiles_list=get_tiles_list();
1753         if (phase == 3)
1754                 create_tile_hash_list(tiles_list);
1755         next=g_list_first(tiles_list);
1756         last=&tile_head_root;
1757         if (! maxlen) {
1758                 while (next) {
1759                         if (strlen(next->data) > maxlen)
1760                                 maxlen=strlen(next->data);
1761                         next=g_list_next(next);
1762                 }
1763         }
1764         len=maxlen;
1765         while (len >= 0) {
1766 #if 0
1767                 fprintf(stderr,"PROGRESS: collecting tiles with len=%d\n", len);
1768 #endif
1769 #ifdef GENERATE_INDEX
1770                 if (! len)
1771                         write_countrydir(phase, maxlen);
1772 #endif
1773                 next=g_list_first(tiles_list);
1774                 while (next) {
1775                         if (strlen(next->data) == len) {
1776                                 th=g_hash_table_lookup(tile_hash, next->data);
1777                                 if (phase == 3) {
1778                                         *last=th;
1779                                         last=&th->next;
1780                                         th->next=NULL;
1781                                         th->zipnum=zipnum;
1782                                         fprintf(out,"%s:%d",(char *)next->data,th->total_size);
1783
1784                                         for ( idx = 0; idx< th->num_subtiles; idx++ ){
1785                         data= th_get_subtile( th, idx );
1786                                                 fprintf(out,":%s", *data);
1787                                         }
1788
1789                                         fprintf(out,"\n");
1790                                 }
1791                                 if (th->name[0])
1792                                         index_submap_add(phase, th, &tiles_list);
1793                                 zipnum++;
1794                                 processed_tiles++;
1795                         }
1796                         next=g_list_next(next);
1797                 }
1798                 len--;
1799         }
1800 }
1801
1802 static void
1803 merge_tiles(void)
1804 {
1805         struct tile_head *th;
1806         char basetile[1024];
1807         char subtile[1024];
1808         GList *tiles_list_sorted,*last;
1809         int i,i_min,len,size_all,size[5],size_min,work_done;
1810         long long zip_size;
1811
1812         do {
1813                 tiles_list_sorted=get_tiles_list();
1814                 fprintf(stderr,"PROGRESS: sorting %d tiles\n", g_list_length(tiles_list_sorted));
1815                 tiles_list_sorted=g_list_sort(tiles_list_sorted, (GCompareFunc)strcmp);
1816                 fprintf(stderr,"PROGRESS: sorting %d tiles done\n", g_list_length(tiles_list_sorted));
1817                 last=g_list_last(tiles_list_sorted);
1818                 zip_size=0;
1819                 while (last) {
1820                         th=g_hash_table_lookup(tile_hash, last->data);
1821                         zip_size+=th->total_size;
1822                         last=g_list_previous(last);
1823                 }
1824                 fprintf(stderr,"DEBUG: size=%Ld\n", zip_size);
1825                 last=g_list_last(tiles_list_sorted);
1826                 work_done=0;
1827                 while (last) {
1828                         processed_tiles++;
1829                         len=strlen(last->data);
1830                         if (len >= 1) {
1831                                 strcpy(basetile,last->data);
1832                                 basetile[len-1]='\0';
1833                                 strcpy(subtile,last->data);
1834                                 for (i = 0 ; i < 4 ; i++) {
1835                                         subtile[len-1]='a'+i;
1836                                         size[i]=tile_data_size(subtile);
1837                                 }
1838                                 size[4]=tile_data_size(basetile);
1839                                 size_all=size[0]+size[1]+size[2]+size[3]+size[4];
1840                                 if (size_all < 65536 && size_all > 0 && size_all != size[4]) {
1841                                         for (i = 0 ; i < 4 ; i++) {
1842                                                 subtile[len-1]='a'+i;
1843                                                 work_done+=merge_tile(basetile, subtile);
1844                                         }
1845                                 } else {
1846                                         for (;;) {
1847                                                 size_min=size_all;
1848                                                 i_min=-1;
1849                                                 for (i = 0 ; i < 4 ; i++) {
1850                                                         if (size[i] && size[i] < size_min) {
1851                                                                 size_min=size[i];
1852                                                                 i_min=i;
1853                                                         }
1854                                                 }
1855                                                 if (i_min == -1)
1856                                                         break;
1857                                                 if (size[4]+size_min >= 65536)
1858                                                         break;
1859                                                 subtile[len-1]='a'+i_min;
1860                                                 work_done+=merge_tile(basetile, subtile);
1861                                                 size[4]+=size[i_min];
1862                                                 size[i_min]=0;
1863                                         }
1864                                 }
1865                         }
1866                         last=g_list_previous(last);
1867                 }
1868                 g_list_free(tiles_list_sorted);
1869                 fprintf(stderr,"PROGRESS: merged %d tiles\n", work_done);
1870         } while (work_done);
1871 }
1872
1873 struct country_index_item {
1874         struct item_bin item;
1875         struct attr_bin attr_country_id;
1876         int country_id;
1877         struct attr_bin attr_zipfile_ref;
1878         int zipfile_ref;
1879 };
1880
1881 static void
1882 index_country_add(int phase, int country_id, int zipnum)
1883 {
1884         struct country_index_item ii;
1885         char *index_tile="";
1886
1887         ii.item.len=sizeof(ii)/4-1;
1888         ii.item.type=type_countryindex;
1889         ii.item.clen=0;
1890
1891         ii.attr_country_id.len=2;
1892         ii.attr_country_id.type=attr_country_id;
1893         ii.country_id=country_id;
1894
1895         ii.attr_zipfile_ref.len=2;
1896         ii.attr_zipfile_ref.type=attr_zipfile_ref;
1897         ii.zipfile_ref=zipnum;
1898
1899         if (phase == 3)
1900                 tile_extend(index_tile, (struct item_bin *)&ii, NULL);
1901         else
1902                 write_item(index_tile, (struct item_bin *)&ii);
1903 }
1904
1905 #ifdef GENERATE_INDEX
1906 struct aux_tile {
1907         char *name;
1908         char *filename;
1909         int size;
1910 };
1911
1912 static int
1913 add_aux_tile(int phase, char *name, char *filename, int size)
1914 {
1915         struct aux_tile *at;
1916         if (phase == 3) {
1917                 at=g_new0(struct aux_tile, 1);
1918                 at->name=g_strdup(name);
1919                 at->filename=g_strdup(filename);
1920                 at->size=size;
1921                 aux_tile_list=g_list_append(aux_tile_list, at);
1922         }
1923         return zipnum++;
1924 }
1925
1926 static int
1927 write_aux_tiles(FILE *out, FILE *dir_out, int compression_level, int namelen)
1928 {
1929         GList *l=aux_tile_list;
1930         struct aux_tile *at;
1931         char *buffer;
1932         FILE *f;
1933         int count=0;
1934
1935         while (l) {
1936                 at=l->data;
1937                 buffer=malloc(at->size);
1938                 assert(buffer != NULL);
1939                 f=fopen(at->filename,"r");
1940                 assert(f != NULL);
1941                 fread(buffer, at->size, 1, f);
1942                 fclose(f);
1943                 write_zipmember(out, dir_out, at->name, namelen, buffer, at->size, compression_level);
1944                 count++;
1945                 l=g_list_next(l);
1946         }
1947         return count;
1948 }
1949
1950 static void
1951 write_countrydir(int phase, int maxnamelen)
1952 {
1953         int i,zipnum;
1954         int max=11;
1955         char tilename[32];
1956         char searchtile[32];
1957         char filename[32];
1958         struct country_table *co;
1959         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1960                 co=&country_table[i];
1961                 if (co->size) {
1962                         tilename[0]='\0';
1963                         tile(&co->r, tilename, max);
1964                         sprintf(searchtile,"%ss%d", tilename, 0);
1965                         sprintf(filename,"country_%d.bin", co->countryid);
1966                         zipnum=add_aux_tile(phase, searchtile, filename, co->size);
1967                         index_country_add(phase,co->countryid,zipnum);
1968                 }
1969         }
1970 }
1971
1972 static void
1973 remove_countryfiles(void)
1974 {
1975         int i;
1976         char filename[32];
1977         struct country_table *co;
1978
1979         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1980                 co=&country_table[i];
1981                 if (co->size) {
1982                         sprintf(filename,"country_%d.bin", co->countryid);
1983                         unlink(filename);
1984                 }
1985         }
1986 }
1987 #endif
1988
1989 static int
1990 phase34(int phase, int maxnamelen, FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
1991 {
1992
1993         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1994         bytes_read=0;
1995         sig_alrm(0);
1996         if (phase == 3)
1997                 tile_hash=g_hash_table_new(g_str_hash, g_str_equal);
1998         if (ways_in)
1999                 phase34_process_file(phase, ways_in);
2000         if (nodes_in)
2001                 phase34_process_file(phase, nodes_in);
2002         fprintf(stderr,"read %d bytes\n", bytes_read);
2003         if (phase == 3)
2004                 merge_tiles();
2005         sig_alrm(0);
2006         alarm(0);
2007         write_tilesdir(phase, maxnamelen, tilesdir_out);
2008
2009         return 0;
2010
2011 }
2012
2013 static int
2014 phase3(FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2015 {
2016         return phase34(3, 0, ways_in, nodes_in, tilesdir_out);
2017 }
2018
2019 static long long zipoffset;
2020 static int zipdir_size;
2021
2022 static int
2023 compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)
2024 {
2025         z_stream stream;
2026         int err;
2027
2028         stream.next_in = (Bytef*)source;
2029         stream.avail_in = (uInt)sourceLen;
2030         stream.next_out = dest;
2031         stream.avail_out = (uInt)*destLen;
2032         if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
2033
2034         stream.zalloc = (alloc_func)0;
2035         stream.zfree = (free_func)0;
2036         stream.opaque = (voidpf)0;
2037
2038         err = deflateInit2(&stream, level, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY);
2039         if (err != Z_OK) return err;
2040
2041         err = deflate(&stream, Z_FINISH);
2042         if (err != Z_STREAM_END) {
2043                 deflateEnd(&stream);
2044                 return err == Z_OK ? Z_BUF_ERROR : err;
2045         }
2046         *destLen = stream.total_out;
2047
2048         err = deflateEnd(&stream);
2049         return err;
2050 }
2051
2052 static void
2053 write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level)
2054 {
2055         struct zip_lfh lfh = {
2056                 0x04034b50,
2057                 0x0a,
2058                 0x0,
2059                 0x0,
2060                 0xbe2a,
2061                 0x5d37,
2062                 0x0,
2063                 0x0,
2064                 0x0,
2065                 filelen,
2066                 0x0,
2067         };
2068         struct zip_cd cd = {
2069                 0x02014b50,
2070                 0x17,
2071                 0x00,
2072                 0x0a,
2073                 0x00,
2074                 0x0000,
2075                 0x0,
2076                 0xbe2a,
2077                 0x5d37,
2078                 0x0,
2079                 0x0,
2080                 0x0,
2081                 filelen,
2082                 0x0000,
2083                 0x0000,
2084                 0x0000,
2085                 0x0000,
2086                 0x0,
2087                 zipoffset,
2088         };
2089         char filename[filelen+1];
2090         int error,crc,len,comp_size=data_size;
2091         uLongf destlen=data_size+data_size/500+12;
2092         char compbuffer[destlen];
2093
2094         crc=crc32(0, NULL, 0);
2095         crc=crc32(crc, (unsigned char *)data, data_size);
2096         if (compression_level) {
2097                 error=compress2_int((Byte *)compbuffer, &destlen, (Bytef *)data, data_size, compression_level);
2098                 if (error == Z_OK) {
2099                         if (destlen < data_size) {
2100                                 data=compbuffer;
2101                                 comp_size=destlen;
2102                         }
2103                 } else {
2104                         fprintf(stderr,"compress2 returned %d\n", error);
2105                 }
2106         }
2107         lfh.zipcrc=crc;
2108         lfh.zipsize=comp_size;
2109         lfh.zipuncmp=data_size;
2110         lfh.zipmthd=compression_level ? 8:0;
2111         cd.zipccrc=crc;
2112         cd.zipcsiz=comp_size;
2113         cd.zipcunc=data_size;
2114         cd.zipcmthd=compression_level ? 8:0;
2115         strcpy(filename, name);
2116         len=strlen(filename);
2117         while (len < filelen) {
2118                 filename[len++]='_';
2119         }
2120         filename[filelen]='\0';
2121         fwrite(&lfh, sizeof(lfh), 1, out);
2122         fwrite(filename, filelen, 1, out);
2123         fwrite(data, comp_size, 1, out);
2124         zipoffset+=sizeof(lfh)+filelen+comp_size;
2125         fwrite(&cd, sizeof(cd), 1, dir_out);
2126         fwrite(filename, filelen, 1, dir_out);
2127         zipdir_size+=sizeof(cd)+filelen;
2128 }
2129
2130 static int
2131 process_slice(FILE *ways_in, FILE *nodes_in, int size, int maxnamelen, FILE *out, FILE *dir_out, int compression_level)
2132 {
2133         struct tile_head *th;
2134         char *slice_data,*zip_data;
2135         int zipfiles=0;
2136
2137         slice_data=malloc(size);
2138         assert(slice_data != NULL);
2139         zip_data=slice_data;
2140         th=tile_head_root;
2141         while (th) {
2142                 if (th->process) {
2143                         th->zip_data=zip_data;
2144                         zip_data+=th->total_size;
2145                 }
2146                 th=th->next;
2147         }
2148         if (ways_in)
2149                 fseek(ways_in, 0, SEEK_SET);
2150         if (nodes_in)
2151                 fseek(nodes_in, 0, SEEK_SET);
2152         phase34(4, maxnamelen, ways_in, nodes_in, NULL);
2153
2154         th=tile_head_root;
2155         while (th) {
2156                 if (th->process) {
2157 #ifdef GENERATE_INDEX
2158                         if (! strlen(th->name))
2159                                 zipfiles+=write_aux_tiles(out, dir_out, compression_level, maxnamelen);
2160 #endif
2161                         if (th->total_size != th->total_size_used) {
2162                                 fprintf(stderr,"Size error '%s': %d vs %d\n", th->name, th->total_size, th->total_size_used);
2163                                 exit(1);
2164                         } else {
2165                                 if (strlen(th->name))
2166                                         write_zipmember(out, dir_out, th->name, maxnamelen, th->zip_data, th->total_size, compression_level);
2167                                 else {
2168                                         write_zipmember(out, dir_out, "index", sizeof("index")-1, th->zip_data, th->total_size, compression_level);
2169                                 }
2170                                 zipfiles++;
2171                         }
2172                 }
2173                 th=th->next;
2174         }
2175         free(slice_data);
2176
2177         return zipfiles;
2178 }
2179
2180 static void
2181 cat(FILE *in, FILE *out)
2182 {
2183         size_t size;
2184         char buffer[4096];
2185         while ((size=fread(buffer, 1, 4096, in)))
2186                 fwrite(buffer, 1, size, out);
2187 }
2188
2189 static int
2190 phase4(FILE *ways_in, FILE *nodes_in, FILE *out, FILE *dir_out, int compression_level)
2191 {
2192         int slice_size=1024*1024*1024;
2193         int maxnamelen,size,slices;
2194         int zipfiles=0;
2195         struct tile_head *th,*th2;
2196         struct zip_eoc eoc = {
2197                 0x06054b50,
2198                 0x0000,
2199                 0x0000,
2200                 0x0000,
2201                 0x0000,
2202                 0x0,
2203                 0x0,
2204                 0x0,
2205         };
2206
2207         maxnamelen=create_tile_hash();
2208
2209         th=tile_head_root;
2210         size=0;
2211         slices=0;
2212         fprintf(stderr, "Maximum slice size %d\n", slice_size);
2213         while (th) {
2214                 if (size + th->total_size > slice_size) {
2215                         fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2216                         size=0;
2217                         slices++;
2218                 }
2219                 size+=th->total_size;
2220                 th=th->next;
2221         }
2222         if (size)
2223                 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2224         th=tile_head_root;
2225         size=0;
2226         slices=0;
2227         while (th) {
2228                 th2=tile_head_root;
2229                 while (th2) {
2230                         th2->process=0;
2231                         th2=th2->next;
2232                 }
2233                 size=0;
2234                 while (th && size+th->total_size < slice_size) {
2235                         size+=th->total_size;
2236                         th->process=1;
2237                         th=th->next;
2238                 }
2239                 zipfiles+=process_slice(ways_in, nodes_in, size, maxnamelen, out, dir_out, compression_level);
2240                 slices++;
2241         }
2242         fseek(dir_out, 0, SEEK_SET);
2243         cat(dir_out, out);
2244         eoc.zipenum=zipfiles;
2245         eoc.zipecenn=zipfiles;
2246         eoc.zipecsz=zipdir_size;
2247         eoc.zipeofst=zipoffset;
2248         fwrite(&eoc, sizeof(eoc), 1, out);
2249         sig_alrm(0);
2250         alarm(0);
2251         return 0;
2252 }
2253
2254 static void
2255 usage(FILE *f)
2256 {
2257         fprintf(f,"\n");
2258         fprintf(f,"osm2navit - parse osm textfile and converts to NavIt binfile format\n\n");
2259         fprintf(f,"Usage :\n");
2260         fprintf(f,"bzcat planet.osm.bz2 | osm2navit mymap.bin\n");
2261         fprintf(f,"Available switches:\n");
2262         fprintf(f,"-h (--help)              : this screen\n");
2263         fprintf(f,"-N (--nodes-only)        : process only nodes\n");
2264         fprintf(f,"-W (--ways-only)         : process only ways\n");
2265         fprintf(f,"-a (--attr-debug-level)  : control which data is included in the debug attribute\n");
2266         fprintf(f,"-c (--dump-coordinates)  : dump coordinates after phase 1\n");
2267         fprintf(f,"-e (--end)               : end at specified phase\n");
2268         fprintf(f,"-k (--keep-tmpfiles)     : do not delete tmp files after processing. useful to reuse them\n\n");
2269         fprintf(f,"-o (--coverage)          : map every street to item overage\n");
2270         fprintf(f,"-s (--start)             : start at specified phase\n");
2271         fprintf(f,"-i (--input-file)        : specify the input file name (OSM), overrules default stdin\n");
2272         fprintf(f,"-w (--dedupe-ways)       : ensure no duplicate ways or nodes. useful when using several input files\n");
2273         fprintf(f,"-z (--compression-level) : set the compression level\n");
2274         exit(1);
2275 }
2276
2277 int main(int argc, char **argv)
2278 {
2279         FILE *ways=NULL,*ways_split=NULL,*nodes=NULL,*tilesdir,*zipdir,*res;
2280         char *map=g_strdup(attrmap);
2281         int c,start=1,end=4,dump_coordinates=0;
2282         int keep_tmpfiles=0;
2283         int process_nodes=1, process_ways=1;
2284         int compression_level=9;
2285         char *result;
2286         FILE* input_file = stdin;
2287
2288
2289         while (1) {
2290 #if 0
2291                 int this_option_optind = optind ? optind : 1;
2292 #endif
2293                 int option_index = 0;
2294                 static struct option long_options[] = {
2295                         {"attr-debug-level", 1, 0, 'a'},
2296                         {"compression-level", 1, 0, 'z'},
2297                         {"coverage", 0, 0, 'o'},
2298                         {"dedupe-ways", 0, 0, 'w'},
2299                         {"end", 1, 0, 'e'},
2300                         {"help", 0, 0, 'h'},
2301                         {"keep-tmpfiles", 0, 0, 'k'},
2302                         {"nodes-only", 0, 0, 'N'},
2303                         {"start", 1, 0, 's'},
2304                         {"input-file", 1, 0, 'i'},
2305                         {"ignore-unknown", 0, 0, 'n'},
2306                         {"ways-only", 0, 0, 'W'},
2307                         {0, 0, 0, 0}
2308                 };
2309                 c = getopt_long (argc, argv, "Nni:Wa:ce:hks:w", long_options, &option_index);
2310                 if (c == -1)
2311                         break;
2312                 switch (c) {
2313                 case 'N':
2314                         process_ways=0;
2315                         break;
2316                 case 'W':
2317                         process_nodes=0;
2318                         break;
2319                 case 'a':
2320                         attr_debug_level=atoi(optarg);
2321                         break;
2322                 case 'c':
2323                         dump_coordinates=1;
2324                         break;
2325                 case 'e':
2326                         end=atoi(optarg);
2327                         break;
2328                 case 'h':
2329                         usage(stdout);
2330                         break;
2331                 case 'n':
2332                         fprintf(stderr,"I will IGNORE unknown types\n");
2333                         ignore_unkown=1;
2334                         break;
2335                 case 'k':
2336                         fprintf(stderr,"I will KEEP tmp files\n");
2337                         keep_tmpfiles=1;
2338                         break;
2339                 case 'o':
2340                         coverage=1;
2341                         break;
2342                 case 's':
2343                         start=atoi(optarg);
2344                         break;
2345                 case 'w':
2346                         dedupe_ways_hash=g_hash_table_new(NULL, NULL);
2347                         break;
2348                 case 'i':
2349                         input_file = fopen( optarg, "r" );
2350                         if ( input_file ==  NULL )
2351                         {
2352                             fprintf( stderr, "\nInput file (%s) not found\n", optarg );
2353                             exit( -1 );
2354                         }
2355                         break;
2356                 case 'z':
2357                         compression_level=atoi(optarg);
2358                         break;
2359                 case '?':
2360                         usage(stderr);
2361                         break;
2362                 default:
2363                         fprintf(stderr,"c=%d\n", c);
2364                 }
2365
2366         }
2367         if (optind != argc-1)
2368                 usage(stderr);
2369         result=argv[optind];
2370         build_attrmap(map);
2371 #ifdef GENERATE_INDEX
2372         build_countrytable();
2373 #endif
2374
2375
2376         if (start == 1) {
2377                 if (process_ways)
2378                         ways=fopen("ways.tmp","wb+");
2379                 if (process_nodes)
2380                         nodes=fopen("nodes.tmp","wb+");
2381                 phase=1;
2382                 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
2383                 phase1(input_file,ways,nodes);
2384                 if (ways)
2385                         fclose(ways);
2386                 if (nodes)
2387                         fclose(nodes);
2388 #ifdef GENERATE_INDEX
2389                 fprintf(stderr,"PROGRESS: Phase 1: sorting countries\n");
2390                 sort_countries();
2391 #endif
2392         }
2393         if (end == 1 || dump_coordinates)
2394                 save_buffer("coords.tmp",&node_buffer);
2395         if (end == 1)
2396                 exit(0);
2397         if (start == 2)
2398                 load_buffer("coords.tmp",&node_buffer);
2399         if (start <= 2) {
2400                 if (process_ways) {
2401                         ways=fopen("ways.tmp","rb");
2402                         ways_split=fopen("ways_split.tmp","wb+");
2403                         phase=2;
2404                         fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
2405                         phase2(ways,ways_split);
2406                         fclose(ways_split);
2407                         fclose(ways);
2408                         if(!keep_tmpfiles)
2409                                 remove("ways.tmp");
2410                 } else
2411                         fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
2412         }
2413         free(node_buffer.base);
2414         node_buffer.base=NULL;
2415         node_buffer.malloced=0;
2416         node_buffer.size=0;
2417         if (end == 2)
2418                 exit(0);
2419         if (start <= 3) {
2420                 phase=3;
2421                 fprintf(stderr,"PROGRESS: Phase 3: generating tiles\n");
2422                 if (process_ways)
2423                         ways_split=fopen("ways_split.tmp","rb");
2424                 if (process_nodes)
2425                         nodes=fopen("nodes.tmp","rb");
2426                 tilesdir=fopen("tilesdir.tmp","wb+");
2427                 phase3(ways_split,nodes,tilesdir);
2428                 fclose(tilesdir);
2429                 if (nodes)
2430                         fclose(nodes);
2431                 if (ways_split)
2432                         fclose(ways_split);
2433         }
2434         if (end == 3)
2435                 exit(0);
2436         if (start <= 4) {
2437                 phase=4;
2438                 fprintf(stderr,"PROGRESS: Phase 4: assembling map\n");
2439                 if (process_ways)
2440                         ways_split=fopen("ways_split.tmp","rb");
2441                 if (process_nodes)
2442                         nodes=fopen("nodes.tmp","rb");
2443                 res=fopen(result,"wb+");
2444                 zipdir=fopen("zipdir.tmp","wb+");
2445                 phase4(ways_split,nodes,res,zipdir,compression_level);
2446                 fclose(zipdir);
2447                 fclose(res);
2448                 if (nodes)
2449                         fclose(nodes);
2450                 if (ways_split)
2451                         fclose(ways_split);
2452                 if(!keep_tmpfiles) {
2453                         remove("nodes.tmp");
2454                         remove("ways_split.tmp");
2455                         remove("tilesdir.tmp");
2456                         remove("zipdir.tmp");
2457 #ifdef GENERATE_INDEX
2458                         remove_countryfiles();
2459 #endif
2460                 }
2461         }
2462         return 0;
2463 }