--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flex.lua Mon Sep 14 22:55:45 2020 +0300 @@ -0,0 +1,1011 @@ +-- This configuration for the flex backend tries to be compatible with the +-- original pgsql c-transform backend. There might be some corner cases but +-- it should mostly do exactly the same. + +-- Set this to true if you were using option -K|--keep-coastlines. +local keep_coastlines = true + +-- Set this to the table name prefix (what used to be option -p|--prefix). +local prefix = 'planet_osm' + +-- Set this to true if multipolygons should be written as polygons into db +-- (what used to be option -G|--multi-geometry). +local multi_geometry = true + +-- Set this to true if you want an hstore column (what used to be option +-- -k|--hstore). Can not be true if "hstore_all" is true. +local hstore = true + +-- Set this to true if you want all tags in an hstore column (what used to +-- be option -j|--hstore-all). Can not be true if "hstore" is true. +local hstore_all = false + +-- Only keep objects that have a value in one of the non-hstore columns +-- (normal action with --hstore is to keep all objects). Equivalent to +-- what used to be set through option --hstore-match-only. +local hstore_match_only = true + +-- Set this to add an additional hstore (key/value) column containing all tags +-- that start with the specified string, eg "name:". Will produce an extra +-- hstore column that contains all "name:xx" tags. Equivalent to what used to +-- be set through option -z|--hstore-column. +local hstore_column = nil + +-- There is some very old specialized handling of route relations in osm2pgsql, +-- which you probably don't need. This is disabled here, but you can enable +-- it by setting this to true. If you don't understand this, leave it alone. +local enable_legacy_route_processing = false + +-- --------------------------------------------------------------------------- + +if hstore and hstore_all then + error("hstore and hstore_all can't be both true") +end + +-- Used for splitting up long linestrings +if osm2pgsql.srid == 4326 then + max_length = 1 +else + max_length = 100000 +end + +-- Ways with any of the following keys will be treated as polygon +local polygon_keys = { + 'aeroway', + 'amenity', + 'building', + 'harbour', + 'historic', + 'landuse', + 'leisure', + 'man_made', + 'military', + 'natural', + 'office', + 'place', + 'power', + 'public_transport', + 'shop', + 'sport', + 'tourism', + 'water', + 'waterway', + 'wetland', + 'abandoned:aeroway', + 'abandoned:amenity', + 'abandoned:building', + 'abandoned:landuse', + 'abandoned:power', + 'area:highway' +} + +-- Objects without any of the following keys will be deleted +local generic_keys = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'area', + 'barrier', + 'bicycle', + 'boundary', + 'brand', + 'bridge', + 'building', + 'capital', + 'construction', + 'covered', + 'culvert', + 'cutting', + 'denomination', + 'disused', + 'ele', + 'embankment', + 'foot', + 'generation:source', + 'harbour', + 'healthcare', + 'highway', + 'historic', + 'hours', + 'intermittent', + 'junction', + 'landuse', + 'layer', + 'leisure', + 'lock', + 'man_made', + 'military', + 'motorcar', + 'name', + 'natural', + 'office', + 'oneway', + 'operator', + 'place', + 'population', + 'power', + 'power_source', + 'public_transport', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'sport', + 'surface', + 'toll', + 'tourism', + 'tower:type', + 'tracktype', + 'tunnel', + 'water', + 'waterway', + 'wetland', + 'width', + 'wood', + 'abandoned:aeroway', + 'abandoned:amenity', + 'abandoned:building', + 'abandoned:landuse', + 'abandoned:power', + 'area:highway' +} + +-- The following keys will be deleted +local delete_keys = { + 'attribution', + 'comment', + 'created_by', + 'fixme', + 'note', + 'note:*', + 'odbl', + 'odbl:note', + 'source', + 'source:*', + 'source_ref', + 'way', + 'way_area', + 'z_order', +} + +local point_columns = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'area', + 'barrier', + 'bicycle', + 'brand', + 'bridge', + 'boundary', + 'building', + 'capital', + 'construction', + 'covered', + 'culvert', + 'cutting', + 'denomination', + 'disused', + 'ele', + 'embankment', + 'foot', + 'generator:source', + 'harbour', + 'highway', + 'historic', + 'horse', + 'intermittent', + 'junction', + 'landuse', + 'layer', + 'leisure', + 'lock', + 'man_made', + 'military', + 'motorcar', + 'name', + 'name:fi', + 'name:sv', + 'minority_name', + 'majority_name', + 'bilingual_name', + 'natural', + 'office', + 'oneway', + 'operator', + 'place', + 'population', + 'power', + 'power_source', + 'public_transport', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'sport', + 'surface', + 'toll', + 'tourism', + 'tower:type', + 'tunnel', + 'water', + 'waterway', + 'wetland', + 'width', + 'wood', +} + +local non_point_columns = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'area', + 'barrier', + 'bicycle', + 'brand', + 'bridge', + 'boundary', + 'building', + 'construction', + 'covered', + 'culvert', + 'cutting', + 'denomination', + 'disused', + 'embankment', + 'foot', + 'generator:source', + 'harbour', + 'highway', + 'historic', + 'horse', + 'intermittent', + 'junction', + 'landuse', + 'layer', + 'leisure', + 'lock', + 'man_made', + 'military', + 'motorcar', + 'name', + 'name:fi', + 'name:sv', + 'minority_name', + 'majority_name', + 'bilingual_name', + 'natural', + 'office', + 'oneway', + 'operator', + 'place', + 'population', + 'power', + 'power_source', + 'public_transport', + 'railway', + 'ref', + 'ref_class', + 'ref_width', + 'religion', + 'route', + 'service', + 'shop', + 'sport', + 'surface', + 'toll', + 'tourism', + 'tower:type', + 'tracktype', + 'tunnel', + 'water', + 'waterway', + 'wetland', + 'width', + 'wood', +} + +-- highlight some suburbs in the capital region +boroughs = { + -- Helsinki + ["Q2116584"] = 1, -- Vuosaari / Nordsjö + ["Q2510635"] = 1, -- Oulunkylä / Åggelby + ["Q3130945"] = 1, -- Viikki / Vik + ["Q1614778"] = 1, -- Herttoniemi / Hertonäs + -- Espoo + ["Q166942"] = 1, -- Leppävaara / Alberga + ["Q3107346"] = 1, -- Matinkylä / Mattby + ["Q211491"] = 1, -- Espoonlahti / Esboviken + ["Q211489"] = 1, -- Espoon keskus / Esbo centrum + ["Q219044"] = 1, -- Kauklahti / Köklax + ["Q1668730"] = 1, -- Tapiola / Hagalund + -- Vantaa + ["Q2640455"] = 1, -- Tikkurila / Dickursby + ["Q4412122"] = 1, -- Aviapolis + ["Q3736737"] = 1, -- Myyrmäki / Myrbacka + ["Q4557316"] = 1, -- Kivistö + ["Q4556551"] = 1, -- Koivukylä / Björkby + ["Q3742144"] = 1, -- Korso + ["Q4556262"] = 1, -- Hakunila / Håkansböle +} + +-- +-- Name in majority language for object. This is name unless both name:fi and +-- name:sv disagree with name. +-- @param tags Raw OSM tags +-- @return Bilingual name +-- +function majority_name(tags) + if tags.name ~= nil and tags['name:sv'] ~= nil and tags['name:fi'] and tags.name ~= tags['name:fi'] and tags.name ~= tags['name:sv'] + then + return tags['name:fi'] + else + return tags.name + end +end +-- +-- Name in minority language for object. This is the Swedish name if name is in +-- Finnish, and the Finnish name if name is in Swedish. +-- @param tags Raw OSM tags +-- @return Bilingual name +-- +function minority_name(tags) + if tags['name:sv'] ~= nil and tags['name'] ~= tags['name:sv'] + then + return tags['name:sv'] + elseif tags['name:fi'] ~= nil and tags['name'] ~= tags['name:fi'] + then + return tags['name:fi'] + else + return nil + end +end + +function bilingual_name(tags) + if tags.majority_name ~= nil and tags.minority_name ~= nil + then + return tags.majority_name .. '\n' .. tags.minority_name + else + return tags.majority_name + end +end + +-- Makes highway classes more sensible +function highway_class(tags) + if string.match(tags.highway, '_link$') + then + if tags.tags.functional_class == 'service' or tags.tags.functional_class == 'unclassified' + then + return tags.tags.functional_class + elseif tags.tags.functional_class ~= nil + then + return tags.tags.functional_class..'_link' + elseif tags.highway == 'motorway_link' + then + return 'trunk_link' + else + return tags.highway + end + elseif tags.highway == 'motorway' + then + if tags.tags.functional_class == 'primary' or tags.tags.functional_class == 'secondary' or tags.tags.functional_class == 'tertiary' + then + return tags.tags.functional_class + elseif tags.tags.functional_class == 'service' + then + return 'service' + else + return 'trunk' + end + else + if tags.highway ~= nil and tags.tags.functional_class ~= nil + then + return tags.tags.functional_class + else + return tags.highway + end + end +end + +function has_shields(tags) + return (tags.highway == 'trunk' or tags.highway == 'primary' or tags.highway == 'secondary' or tags.highway == 'tertiary') +end + +-- https://stackoverflow.com/a/7615129 +function mysplit(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + table.insert(t, str) + end + return t +end + +function combine_stringlists(a, b) + if a ~= nil and a ~= '' + then + if b ~= nil and b ~= '' + then + return a..';'..b + else + return a + end + else + return b + end +end + +function combine_refs(a, b) + return combine_stringlists(a, b) +end + +-- Widths of ref characters in Liikenne font, used to calculate shield sizes +character_widths = { + ['0'] = 616, + ['1'] = 337, + ['2'] = 600, + ['3'] = 621, + ['4'] = 689, + ['5'] = 623, + ['6'] = 566, + ['7'] = 499, + ['8'] = 616, + ['9'] = 566, + [' '] = 260, + ['E'] = 613, +} + +function ref_width(ref) + local result = 0 + for i = 1, #ref + do + result = result + (character_widths[ref:sub(i, i)] or 0) + end + return math.max(math.ceil((result - 1000) / 77), 1) +end + +function ref_widths(refs) + local result = '' + for _, ref in ipairs(mysplit(refs, ';')) + do + result = combine_stringlists(result, ref_width(ref)) + end + return result +end + +-- Makes refs more sensible. +-- 5-digit refs are not found on the ground so they are omitted. +-- int_ref is added to refs. +function highway_refs(tags) + if has_shields(tags) and (tags.ref ~= nil or tags.tags.int_ref ~= nil) + then + if tags.junction == 'roundabout' + then + return nil + else + local result = '' + local all_refs = combine_refs(tags.ref, tags.tags.int_ref) + for _, ref in ipairs(mysplit(all_refs, ';')) + do + if tonumber(ref) == nil or tonumber(ref) < 10000 + then + ref = string.gsub(ref, '^E 0', 'E ') + result = combine_refs(result, ref) + end + end + return result + end + else + return tags.ref + end +end + +-- classify road type by number +function road_class_by_ref(ref) + local n = tonumber(ref) + if n ~= nil and n > 0 + then + if n < 40 + then + return 'valtatie' + elseif n < 100 + then + return 'kantatie' + elseif n < 1000 + then + return 'seututie' + else + return 'yhdystie' + end + elseif string.match(ref, '^E ?%d+$') + then + return 'eurooppatie' + else + return 'unknown' + end +end + +function classify_road_numbers(refs) + local result = '' + for _, ref in ipairs(mysplit(refs, ';')) + do + result = combine_stringlists(result, road_class_by_ref(ref)) + end + return result +end + +function gen_columns(text_columns, with_hstore, area, geometry_type) + columns = {} + + local add_column = function (name, type) + columns[#columns + 1] = { column = name, type = type } + end + + for _, c in ipairs(text_columns) do + add_column(c, 'text') + end + + add_column('z_order', 'int') + + if area ~= nil then + if area then + add_column('way_area', 'area') + else + add_column('way_area', 'real') + end + end + + if hstore_column then + add_column(hstore_column, 'hstore') + end + + if with_hstore then + add_column('tags', 'hstore') + end + + add_column('way', geometry_type) + + return columns +end + +local tables = {} + +tables.point = osm2pgsql.define_table{ + name = prefix .. '_point', + ids = { type = 'node', id_column = 'osm_id' }, + columns = gen_columns(point_columns, hstore or hstore_all, nil, 'point') +} + +tables.line = osm2pgsql.define_table{ + name = prefix .. '_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring') +} + +tables.polygon = osm2pgsql.define_table{ + name = prefix .. '_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = gen_columns(non_point_columns, hstore or hstore_all, true, 'geometry') +} + +tables.roads = osm2pgsql.define_table{ + name = prefix .. '_roads', + ids = { type = 'way', id_column = 'osm_id' }, + columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring') +} + +local z_order_lookup = { + proposed = {1, false}, + construction = {2, false}, + steps = {10, false}, + cycleway = {10, false}, + bridleway = {10, false}, + footway = {10, false}, + path = {10, false}, + track = {11, false}, + service = {15, false}, + + tertiary_link = {24, false}, + secondary_link = {25, true}, + primary_link = {27, true}, + trunk_link = {28, true}, + motorway_link = {29, true}, + + raceway = {30, false}, + pedestrian = {31, false}, + living_street = {32, false}, + road = {33, false}, + unclassified = {33, false}, + residential = {33, false}, + tertiary = {34, false}, + secondary = {36, true}, + primary = {37, true}, + trunk = {38, true}, + motorway = {39, true} +} + +function as_bool(value) + return value == 'yes' or value == 'true' or value == '1' +end + +function get_z_order(tags) + local z_order = 100 * math.floor(tonumber(tags.layer or '0') or 0) + local roads = false + + local highway = tags['highway'] + if highway then + local r = z_order_lookup[highway] or {0, false} + z_order = z_order + r[1] + roads = r[2] + end + + if tags.railway then + z_order = z_order + 35 + roads = true + end + + if tags.boundary and tags.boundary == 'administrative' then + roads = true + end + + if as_bool(tags.bridge) then + z_order = z_order + 100 + end + + if as_bool(tags.tunnel) then + z_order = z_order - 100 + end + + return z_order, roads +end + +function make_check_in_list_func(list) + local h = {} + for _, k in ipairs(list) do + h[k] = true + end + return function(tags) + for k, _ in pairs(tags) do + if h[k] then + return true + end + end + return false + end +end + +local is_polygon = make_check_in_list_func(polygon_keys) +local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys) + +function make_column_hash(columns) + local h = {} + + for _, k in ipairs(columns) do + h[k] = true + end + + return h +end + +function make_get_output(columns, hstore_all) + local h = make_column_hash(columns) + if hstore_all then + return function(tags) + local output = {} + local hstore_entries = {} + + for k, _ in pairs(tags) do + if h[k] then + output[k] = tags[k] + end + hstore_entries[k] = tags[k] + end + + return output, hstore_entries + end + else + return function(tags) + local output = {} + local hstore_entries = {} + + for k, _ in pairs(tags) do + if h[k] then + output[k] = tags[k] + else + hstore_entries[k] = tags[k] + end + end + + return output, hstore_entries + end + end +end + +local has_generic_tag = make_check_in_list_func(generic_keys) + +local get_point_output = make_get_output(point_columns, hstore_all) +local get_non_point_output = make_get_output(non_point_columns, hstore_all) + +function get_hstore_column(tags) + local len = #hstore_column + local h = {} + for k, v in pairs(tags) do + if k:sub(1, len) == hstore_column then + h[k:sub(len + 1)] = v + end + end + + if next(h) then + return h + end + return nil +end + +function add_generic_tags(object, output) + output.majority_name = majority_name(output) + output.minority_name = minority_name(output) + output.bilingual_name = bilingual_name(output) + return output +end + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + if object.tags.layer then + object.tags.layer = tonumber(object.tags.layer) + end + + local output + local output_hstore = {} + if hstore or hstore_all then + output, output_hstore = get_point_output(object.tags) + if not next(output) and not next(output_hstore) then + return + end + if hstore_match_only and not has_generic_tag(object.tags) then + return + end + else + output = object.tags + if not has_generic_tag(object.tags) then + return + end + end + + output = add_generic_tags(object, output) + output.tags = output_hstore + + if hstore_column then + output[hstore_column] = get_hstore_column(object.tags) + end + + tables.point:add_row(output) +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + if object.tags.layer then + object.tags.layer = tonumber(object.tags.layer) + end + + + local add_area = false + if object.tags.natural == 'coastline' then + add_area = true + if not keep_coastlines then + object.tags.natural = nil + end + end + + local output + local output_hstore = {} + if hstore or hstore_all then + output, output_hstore = get_non_point_output(object.tags) + if not next(output) and not next(output_hstore) then + return + end + if hstore_match_only and not has_generic_tag(object.tags) then + return + end + if add_area and hstore_all then + output_hstore.area = 'yes' + end + else + output = object.tags + if not has_generic_tag(object.tags) then + return + end + end + + local polygon + local area_tag = object.tags.area + if area_tag == 'yes' or area_tag == '1' or area_tag == 'true' then + polygon = true + elseif area_tag == 'no' or area_tag == '0' or area_tag == 'false' then + polygon = false + else + polygon = is_polygon(object.tags) + end + + if add_area then + output.area = 'yes' + polygon = true + end + + local z_order, roads = get_z_order(object.tags) + output.z_order = z_order + output = add_generic_tags(object, output) + + output.tags = output_hstore + + if output.highway ~= nil + then + if output.highway == 'construction' + then + return + end + output.highway = highway_class(output) + output.ref = highway_refs(output) + if output.highway == 'construction' and output.construction == 'motorway' + then + output.construction = 'trunk' + elseif output.highway == 'construction' and output.construction == 'motorway_link' + then + output.construction = 'trunk_link' + end + + -- classify references for for Finland shield rendering + if output.ref ~= nil + then + output.ref_class = classify_road_numbers(output.ref) + output.ref_width = ref_widths(output.ref) + end + end + + if hstore_column then + output[hstore_column] = get_hstore_column(object.tags) + end + + if polygon and object.is_closed then + output.way = { create = 'area' } + tables.polygon:add_row(output) + else + output.way = { create = 'line', split_at = max_length } + tables.line:add_row(output) + if roads then + tables.roads:add_row(output) + end + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + local type = object.tags.type + if (type ~= 'route') and (type ~= 'multipolygon') and (type ~= 'boundary') then + return + end + object.tags.type = nil + + local output + local output_hstore = {} + if hstore or hstore_all then + output, output_hstore = get_non_point_output(object.tags) + if not next(output) and not next(output_hstore) then + return + end + if hstore_match_only and not has_generic_tag(object.tags) then + return + end + else + output = object.tags + if not has_generic_tag(object.tags) then + return + end + end + + if not next(output) and not next(output_hstore) then + return + end + + if enable_legacy_route_processing and (hstore or hstore_all) and type == 'route' then + if not object.tags.route_name then + output_hstore.route_name = object.tags.name + end + + local state = object.tags.state + if state ~= 'alternate' and state ~= 'connection' then + state = 'yes' + end + + local network = object.tags.network + if network == 'lcn' then + output_hstore.lcn = output_hstore.lcn or state + output_hstore.lcn_ref = output_hstore.lcn_ref or object.tags.ref + elseif network == 'rcn' then + output_hstore.rcn = output_hstore.rcn or state + output_hstore.rcn_ref = output_hstore.rcn_ref or object.tags.ref + elseif network == 'ncn' then + output_hstore.ncn = output_hstore.ncn or state + output_hstore.ncn_ref = output_hstore.ncn_ref or object.tags.ref + elseif network == 'lwn' then + output_hstore.lwn = output_hstore.lwn or state + output_hstore.lwn_ref = output_hstore.lwn_ref or object.tags.ref + elseif network == 'rwn' then + output_hstore.rwn = output_hstore.rwn or state + output_hstore.rwn_ref = output_hstore.rwn_ref or object.tags.ref + elseif network == 'nwn' then + output_hstore.nwn = output_hstore.nwn or state + output_hstore.nwn_ref = output_hstore.nwn_ref or object.tags.ref + end + + local pc = object.tags.preferred_color + if pc == '0' or pc == '1' or pc == '2' or pc == '3' or pc == '4' then + output_hstore.route_pref_color = pc + else + output_hstore.route_pref_color = '0' + end + end + + local make_boundary = false + local make_polygon = false + if type == 'boundary' then + make_boundary = true + elseif type == 'multipolygon' and object.tags.boundary then + make_boundary = true + elseif type == 'multipolygon' then + make_polygon = true + end + + local z_order, roads = get_z_order(object.tags) + output.z_order = z_order + output = add_generic_tags(object, output) + + output.tags = output_hstore + + if hstore_column then + output[hstore_column] = get_hstore_column(object.tags) + end + + if not make_polygon then + output.way = { create = 'line', split_at = max_length } + tables.line:add_row(output) + if roads then + tables.roads:add_row(output) + end + end + + if make_boundary or make_polygon then + output.way = { create = 'area', multi = multi_geometry } + tables.polygon:add_row(output) + end +end +