Generated: Sun Aug 21 11:11:17 2011 from osm-load.pl 2011/03/05 21.8 KB.
#!/usr/bin/perl -w # NAME: osm-load.pl # AIM: Load an XML OSM file use strict; use warnings; use File::Basename; # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] ) use Cwd; my $perl_dir = 'C:\GTools\perl'; unshift(@INC, $perl_dir); #require 'logfile.pl' or die "Unable to load logfile.pl ...\n"; require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl'! Check location and \@INC content.\n"; require 'fg_wsg84.pl' or die "Unable to load fg_wsg84.pl ...\n"; require "Bucket2.pm" or die "Unable to load Bucket2.pm ...\n"; # log file stuff our ($LF); my $pgmname = $0; if ($pgmname =~ /(\\|\/)/) { my @tmpsp = split(/(\\|\/)/,$pgmname); $pgmname = $tmpsp[-1]; } my $outfile = $perl_dir."\\temp.$pgmname.txt"; open_log($outfile); # user variables my $load_log = 1; my $in_file = ''; my $SG_METER_TO_NM = 0.0005399568034557235; my $temp_poly = $perl_dir."\\temppoly."; my $poly_count = 0; # standard fields to poulate in Tags my @standardFields = ('highway','junction','cycleway','tracktype','waterway','railway', 'aeroway','aerialway','power','man_made','leisure', 'amenity','shop','tourism','historic','landuse', 'military','natural','route','boundary','sport', 'abutters','fenced','lit','width','lanes', 'bridge','tunnel','cutting','embankment','layer', 'surface','name','int_name','nat_name','reg_name', 'loc_name','old_name','ref','int_ref','nat_ref', 'reg_ref','loc_ref','old_ref','ncn_ref','place', 'place_name','place_numbers','postal_code','is_in','note','class'); sub in_standard($) { my $tag = shift; my ($tt); foreach $tt (@standardFields) { return 1 if ($tt eq $tag); } return 0; } #Tags with these keys will not be loaded, features with *only* these keys will not be loaded my @ignoreFields = ('created_by','source','converted_by'); #this flag controls whether features with only non standard tags are loaded. my $loadNonstandardTags = 0; my $debug_on = 1; my $def_file = 'C:\Documents and Settings\Geoff McLane\My Documents\FG\OSM\YGIL-map.osm'; ### program variables my @warnings = (); my $cwd = cwd(); my $os = $^O; my $g_minlat = 200; my $g_maxlat = -200; my $g_minlon = 200; my $g_maxlon = -200; my ($g_center_lat,$g_center_lon); my ($g_lat_span,$g_lon_span); my ($g_width,$g_height); sub show_warnings($) { my ($val) = @_; if (@warnings) { prt( "\nGot ".scalar @warnings." WARNINGS...\n" ); foreach my $itm (@warnings) { prt("$itm\n"); } prt("\n"); } else { # prt( "\nNo warnings issued.\n\n" ); } } sub pgm_exit($$) { my ($val,$msg) = @_; if (length($msg)) { $msg .= "\n" if (!($msg =~ /\n$/)); prt($msg); } show_warnings($val); close_log($outfile,$load_log); exit($val); } sub prtw($) { my ($tx) = shift; $tx =~ s/\n$//; prt("$tx\n"); push(@warnings,$tx); } #--------------------------------------------------------------------------- #Gets the XML element name from the string passed in #an end of element tag is /element #--------------------------------------------------------------------------- sub getElement($) { my $line = shift; my $el = ''; my $s = index($line,'<'); if ($s > 0) { my $e = index($line,' ',$s); if ($e > 0) { $el = substr($line,$s+1,$e-$s-1); } else { $e = index($line,'>',$s); if ($e > 0) { $el = substr($line,$s+1,$e-$s-1); $el =~ s/\/$//; } else { prtw("WARNING: NO ELEMENT [$line]?\n"); } } } return $el; } #--------------------------------------------------------------------------- #Gets the value of the named attribute from the string #--------------------------------------------------------------------------- sub getAttributeValue($$) { my ($name,$line) = @_; my $attr = ''; my $sa = index($line,' '.$name.'="'); if ($sa > 0) { #prt("Begin = $sa, "); $sa += length($name) + 3; # past the first '"' my $ea = index($line,'"',$sa); if ($ea > 0) { #prt("Begin = $sa, End = $ea\n"); $attr = substr($line,$sa,$ea - $sa); } } return $attr; } #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- #Extract Node attribute details from a line of xml text #--------------------------------------------------------------------------- sub returnNode($) { my $line = shift; my $nid = getAttributeValue('id',$line); my $nx = getAttributeValue('lon',$line); my $ny = getAttributeValue('lat',$line); return($nid,$nx,$ny); } #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- #extract segment attributes from a line of xml text #--------------------------------------------------------------------------- sub returnSegment($) { my $line = shift; my $sid = getAttributeValue('id',$line); my $sn = getAttributeValue('from',$line); my $en = getAttributeValue('to',$line); return($sid,$sn,$en); } #--------------------------------------------------------------------------- #get the id attribute from a line of xml text #used for ways and its segs, as id is only attribute needed sub returnID($) { my $line = shift; return getAttributeValue('id',$line); } sub returnTags($) { my $line = shift; my $k = getAttributeValue('k',$line); # .lstrip().encode("Latin-1","replace")[:29] my $v = getAttributeValue('v',$line); # .encode("Latin-1","replace")[:254] return ($k,$v); } #--------------------------------------------------------------------------- sub in_world_range($$) { my ($lat,$lon) = @_; return 0 if ($lat < -90); return 0 if ($lat > 90); return 0 if ($lon < -180); return 0 if ($lon > 180); return 1; } #node=('ID','x','y') #segment=('id','start','end') #way=('id','seg1 seg2') #tag=('key','value') sub process_in_file($) { my ($inf) = @_; if (! open INF, "<$inf") { pgm_exit(1,"ERROR: Unable to open file [$inf]\n"); } my @lines = <INF>; close INF; my $lncnt = scalar @lines; prt("Processing $lncnt lines, from [$inf]...\n"); my ($line,$inc,$lnn); my ($element,$ftype,$id,$lon,$lat,$v,$k,$val); my ($minlat,$minlon,$maxlat,$maxlon,$hadbounds,$msg); my %nodes = (); my %node_tags = (); my %way_tags = (); my %way_nd = (); my @NODES = (); my @ftags = (); my @unbuiltways = (); my @ways = (); my @nd_array = (); $lnn = 0; $ftype = -1; $hadbounds = 0; my $hasvalidtags = 0; my $nodecount = 0; my $waycount = 0; my $way = 0; my $waytagcount = 0; my $nodetagcount = 0; my $taggednodecount = 0; foreach $line (@lines) { chomp $line; $lnn++; $element = getElement($line); next if (length($element) == 0); if ($element eq 'node') { $ftype = -1; @ftags = (); ($id,$lon,$lat) = returnNode($line); #if float(node[1])>=-180 and float(node[1])<=180 and float(node[2])>=-90 and float(node[2])<=90: #if ( ($lon >= -180) && ($lon <= 180) && ($lat >= -90) && ($lat <= 90) && ($id > 0)) { if (in_world_range($lat,$lon)) { $ftype = 0; # set a VALID 'node' #nodefile.write(str(node[0])+':'+str(node[1])+':'+str(node[2])+'\n') push(@NODES, [$lon,$lat]); # <bounds minlat="-31.7130540" minlon="148.6136050" maxlat="-31.6947200" maxlon="148.6567780"/> if (defined $nodes{$id}) { prtw("WARNING:$lnn: Node ID [$id] REPEATED\n"); } else { $nodes{$id} = [$lon,$lat]; } # oops, found items CAN be out of this range - so no range check #if ($hadbounds && (($lat < $minlat)||($lat > $maxlat)||($lon < $minlon)||($lon > $maxlon))) { # $msg = "lat,lon $lat,$lon OUT OF BOUNDS "; # prtw("WARNING:$lnn: $msg\n"); #} # but keep the NEW ranges $g_minlat = $lat if ($lat < $g_minlat); $g_maxlat = $lat if ($lat > $g_maxlat); $g_minlon = $lon if ($lon < $g_minlon); $g_maxlon = $lon if ($lon > $g_maxlon); $nodecount++; } else { prtw("WARNING:$lnn: Line $id,$lon,$lat [".substr($line,0,70)."...] FAILED!\n"); } } elsif ($element eq 'way') { # <way id="37623654" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812676" timestamp="2009-07-13T00:57:47Z"> @ftags = (); $ftype = 2; $waycount++; $way = returnID($line); # set the way if (length($way)) { push(@ways,$way); } else { prtw("WARNING:$lnn: way element without ID [$way] line [$line]\n"); } # unbuiltways.write('\n'+str(way[0])+'#') } elsif ($element eq 'nd') { $val = getAttributeValue('ref',$line); if (length($val)) { push(@nd_array,$val); } else { prtw("WARNING:$lnn: nd element without REF [$val] line [$line]\n"); } # unbuiltways.write(str(getAttributeValue('ref',line))+':') } elsif ($element eq 'tag' ) { ($v,$k) = returnTags($line); # prt("tag element: v=$v k=$k ftype=$ftype\n"); if ($ftype == 0) { # tagged node #ignore less useful tags, and if not a standard tag #remove tags with blank values too. lots of wierd keys have blank values if ((length($v) == 0)||(length($k) == 0)) { prtw("WARNING:$lnn:1: ftype=$ftype v=$v, k=$k\n"); }else { #if tag[0] in standardFields and tag[1] !='': if (in_standard($v)) { #ftags.append((tag[0],tag[1])) push(@ftags, [$v,$k]); $hasvalidtags = 1; $nodetagcount++; } elsif ($loadNonstandardTags) { # if tag[0] not in ignoreFields and tag[0] not in standardFields and tag[1] !='': # hasvalidtags=True # row=othernodetagcursor.newrow() # row.Node_ID=long(node[0]) # row.Tag_Name=str(tag[0]) # row.Tag_Value=str(tag[1]) # othernodetagcursor.insertrow(row) # nodetagcount+=1 } } } elsif ($ftype == 2) { #way tags, loading all these except ignorefields and blank valued ones. if ((length($v) == 0)||(length($k) == 0)) { prtw("WARNING:$lnn:2: ftype=$ftype v=$v, k=$k\n"); } else { # if tag[0] in standardFields and tag[1] !='': if ( in_standard($v) ) { push(@ftags, [$v,$k]); #ftags.append((tag[0],tag[1])) $hasvalidtags = 1; $waytagcount++; } elsif ($loadNonstandardTags) { #if tag[0] not in ignoreFields and tag[0] not in standardFields and tag[1] !='': # row=otherwaytagcursor.newrow() # row.Way_ID=long(way[0]) # row.Tag_Name=str(tag[0]) # row.Tag_Value=str(tag[1]) # otherwaytagcursor.insertrow(row) # waytagcount+=1 } } } else { prtw("WARNING:$lnn:3: ftype=$ftype v=$v, k=$k\n"); } } elsif (($element eq '/node') && $hasvalidtags && ($ftype == 0)) { # <node id="441007981" lat="-31.7088281" lon="148.6651552" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812676" timestamp="2009-07-13T00:57:45Z"/> # <node id="441013712" lat="-31.7020985" lon="148.6473310" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812736" timestamp="2009-07-13T01:35:23Z"> # <tag k="railway" v="level_crossing"/> # </node> if (defined $node_tags{$id}) { prtw("WARNING:$lnn: Duplicated ID in node_tags [$id]\n"); } my @a1 = @ftags; $node_tags{$id} = \@a1; @ftags = (); #done with node lets load its shape # frow = nodecursor.newrow() #nodepnt.x=float(node[1]) #nodepnt.y=float(node[2]) # frow.setValue("Node_ID",node[0].encode("Latin-1","replace")) # for f in standardFields: # frow.setValue(f,'') # for sTag in ftags: # frow.setValue(sTag[0],str(sTag[1])) # frow.SetValue('shape', nodepnt) #Load the shape # nodecursor.insertrow(frow) $taggednodecount++; $hasvalidtags = 0; #$ftype = -1; } elsif (($element eq '/way') && $hasvalidtags) { # <way id="30215921" user="VK1RE" uid="69499" visible="true" version="2" changeset="808028" timestamp="2009-01-18T11:05:50Z"> # <nd ref="332988044"/> # <nd ref="332988046"/> # <tag k="created_by" v="Potlatch 0.10f"/> # <tag k="highway" v="residential"/> # </way> #done with way lets load attributes, shape comes later my @a2 = @ftags; if (defined $way_tags{$way}) { prtw("WARNING:$lnn: Duplicated ID in way_tags [$way]\n"); } $way_tags{$way} = \@a2; # trow=waytagcursor.newrow() # trow.SetValue("Way_ID",long(way[0])) # for f in standardFields: # trow.setValue(f,'') # for sTag in ftags: # trow.setValue(sTag[0],str(sTag[1])) # waytagcursor.insertrow(trow) if (defined $way_nd{$way}) { prtw("WARNING:$lnn: Duplicated ID in way_nd [$way]\n"); } my @a3 = @nd_array; $way_nd{$way} = \@a3; @nd_array = (); @ftags = (); $hasvalidtags = 0; #$ftype = -1; } elsif ($element eq 'member') { # } elsif ($element eq 'bounds') { # <bounds minlat="-31.7130540" minlon="148.6136050" maxlat="-31.6947200" maxlon="148.6567780"/> $hadbounds = 1; $minlat = getAttributeValue('minlat',$line); $minlon = getAttributeValue('minlon',$line); $maxlat = getAttributeValue('maxlat',$line); $maxlon = getAttributeValue('maxlon',$line); prt("bounds $minlat,$minlon $maxlat,$maxlon\n"); } elsif ($element eq 'relation') { # } elsif ($element eq '/relation') { # } else { prtw("WARNING:$lnn: Element [$element] NOT dealt with\n"); } } prt( "Verticies=$nodecount, ntc=$nodetagcount, way=$waycount, waytc=$waytagcount, tnc=$taggednodecount\n"); prt("tag bounds $minlat,$minlon $maxlat,$maxlon\n"); $g_center_lat = ($g_minlat + $g_maxlat) / 2; $g_center_lon = ($g_minlon + $g_maxlon) / 2; $g_lat_span = abs($g_maxlat - $g_minlat); $g_lon_span = abs($g_maxlon - $g_minlon); $g_width = int($g_lon_span * 1000)+1; $g_height = int($g_lat_span * 1000)+1; prt("glob bounds $g_minlat,$g_minlon $g_maxlat,$g_maxlon\n"); prt("Center $g_center_lat,$g_center_lon, span $g_lat_span,$g_lon_span h=$g_height w=$g_width\n"); my %hash = (); $hash{'nodes_h'} = \%nodes; $hash{'node_tags_h'} = \%node_tags; $hash{'way_tags_h'} = \%way_tags; $hash{'way_nd_h'} = \%way_nd; $hash{'NODE_a'} = \@NODES; $hash{'unbuiltways_a'} = \@unbuiltways; $hash{'ways_a'} = \@ways; return \%hash; } sub set_decimal1_stg($) { my $r = shift; ${$r} = int((${$r} + 0.05) * 10) / 10; ${$r} = "0.0" if (${$r} == 0); ${$r} .= ".0" if !(${$r} =~ /\./); } sub set_decimal2_stg($) { my $r = shift; ${$r} = int((${$r} + 0.005) * 100) / 100; ${$r} = "0.00" if (${$r} == 0); ${$r} .= ".00" if !(${$r} =~ /\./); } sub set_decimal3_stg($) { my $r = shift; ${$r} = int((${$r} + 0.0005) * 1000) / 1000; ${$r} = "0.000" if (${$r} == 0); ${$r} .= ".000" if !(${$r} =~ /\./); } sub set_int_stg($) { my $r = shift; ${$r} = int(${$r} + 0.5); } sub get_dist_stg_nm($) { my ($dist) = @_; my $nm = $dist * $SG_METER_TO_NM; if ($nm < 1) { set_decimal3_stg(\$nm); } elsif ($nm < 10) { set_decimal2_stg(\$nm); } else { set_decimal1_stg(\$nm); } $nm .= "nm"; return $nm; } sub show_ref_hash($) { my ($rh) = @_; my ($key,$val,$way); my ($rtags,$rnds); my ($tcnt,$ncnt,$i,$v,$k,$id); my ($lat,$lon,$msg,$rll,$cnt,$poly,$tmp); foreach $key (keys %{$rh}) { $val = ${$rh}{$key}; prt("$key "); } prt("\n"); my $rnodes_h = ${$rh}{'nodes_h'}; my $rnode_tags_h = ${$rh}{'node_tags_h'}; my $rway_tags_h = ${$rh}{'way_tags_h'}; my $rway_nd_h = ${$rh}{'way_nd_h'}; my $rnode_a = ${$rh}{'NODE_a'}; my $rubw_a = ${$rh}{'unbuiltways_a'}; my $ways_a = ${$rh}{'ways_a'}; $ncnt = scalar keys(%{$rnodes_h}); prt("Got $ncnt reference points...\n"); foreach $way (keys %{$rway_tags_h}) { if (defined ${$rway_nd_h}{$way}) { $poly = "#2D\n"; # is 2D vector $rtags = ${$rway_tags_h}{$way}; $rnds = ${$rway_nd_h}{$way}; $tcnt = scalar @{$rtags}; $ncnt = scalar @{$rnds}; prt("\nway $way "); for ($i = 0; $i < $tcnt; $i++) { $v = ${$rtags}[$i][0]; $k = ${$rtags}[$i][1]; prt("$v=$k "); $poly .= "$v\n" if ($i == 0); # add TYPE } $poly .= "1\n"; # add only 1 per file $poly .= "$ncnt\n"; # number of entries $poly .= "0\n"; # whole flag prt("$ncnt nd points\n"); # $nodes{$id} = [$lon,$lat]; $msg = ''; $cnt = 0; $i = 0; my ($llat,$llon,$az1,$az2,$dist); my $tot_dist = 0; foreach $id (@{$rnds}) { $i++; if (defined ${$rnodes_h}{$id}) { $rll = ${$rnodes_h}{$id}; $lon = ${$rll}[0]; $lat = ${$rll}[1]; $poly .= sprintf("%.15f %.15f\n", $lon, $lat); if ($i > 1) { fg_geo_inverse_wgs_84 ($llat,$llon,$lat,$lon,\$az1,\$az2,\$dist); $tot_dist += $dist; } $llat = $lat; $llon = $lon; $lon -= $g_center_lon; $lat -= $g_center_lat; $msg .= "$lat,$lon "; $cnt++; if (($cnt == 3)&&($i < $ncnt)) { $msg .= "\n"; $cnt = 0; } } else { prtw("WARNING: No REFERNCE point for ID [$id]\n"); } } $dist = get_dist_stg_nm($tot_dist); prt("Points: $msg $dist\n"); $poly_count++; $tmp = $temp_poly.$poly_count; write2file($poly,$tmp); } else { prtw("WARNING: ID $way in way tags, NOT in way_nd\n"); } } } ######################################### ### MAIN ### parse_args(@ARGV); prt( "$pgmname: in [$cwd]: Hello, World...\n" ); show_ref_hash(process_in_file($in_file)); pgm_exit(0,"Normal exit(0)"); ######################################## sub give_help { prt("$pgmname: version 0.0.1 2010-09-11\n"); prt("Usage: $pgmname [options] in-file\n"); prt("Options:\n"); prt(" --help (-h or -?) = This help, and exit 0.\n"); } sub need_arg { my ($arg,@av) = @_; pgm_exit(1,"ERROR: [$arg] must have following argument!\n") if (!@av); } sub parse_args { my (@av) = @_; my ($arg,$sarg); while (@av) { $arg = $av[0]; if ($arg =~ /^-/) { $sarg = substr($arg,1); $sarg = substr($sarg,1) while ($sarg =~ /^-/); if (($sarg =~ /^h/i)||($sarg eq '?')) { give_help(); pgm_exit(0,"Help exit(0)"); } else { pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n"); } } else { $in_file = $arg; prt("Set input to [$in_file]\n"); } shift @av; } if ((length($in_file) == 0) && $debug_on) { $in_file = $def_file; } if (length($in_file) == 0) { pgm_exit(1,"ERROR: No input files found in command!\n"); } if (! -f $in_file) { pgm_exit(1,"ERROR: Unable to find in file [$in_file]! Check name, location...\n"); } } # eof - template.pl