flex.lua

-- This configuration for the flex backend tries to be compatible with the
-- original pgsql c-transform backend. There might be some corner cases but
-- it should mostly do exactly the same.

-- Set this to true if you were using option -K|--keep-coastlines.
local keep_coastlines = true

-- Set this to the table name prefix (what used to be option -p|--prefix).
local prefix = 'planet_osm'

-- Set this to true if multipolygons should be written as polygons into db
-- (what used to be option -G|--multi-geometry).
local multi_geometry = true

-- Set this to true if you want an hstore column (what used to be option
-- -k|--hstore). Can not be true if "hstore_all" is true.
local hstore = true

-- Set this to true if you want all tags in an hstore column (what used to
-- be option -j|--hstore-all). Can not be true if "hstore" is true.
local hstore_all = false

-- Only keep objects that have a value in one of the non-hstore columns
-- (normal action with --hstore is to keep all objects). Equivalent to
-- what used to be set through option --hstore-match-only.
local hstore_match_only = true

-- Set this to add an additional hstore (key/value) column containing all tags
-- that start with the specified string, eg "name:". Will produce an extra
-- hstore column that contains all "name:xx" tags. Equivalent to what used to
-- be set through option -z|--hstore-column.
local hstore_column = nil

-- There is some very old specialized handling of route relations in osm2pgsql,
-- which you probably don't need. This is disabled here, but you can enable
-- it by setting this to true. If you don't understand this, leave it alone.
local enable_legacy_route_processing = false

-- ---------------------------------------------------------------------------

if hstore and hstore_all then
    error("hstore and hstore_all can't be both true")
end

-- Used for splitting up long linestrings
if osm2pgsql.srid == 4326 then
    max_length = 1
else
    max_length = 100000
end

-- Ways with any of the following keys will be treated as polygon
local polygon_keys = {
    'aeroway',
    'amenity',
    'building',
    'harbour',
    'historic',
    'landuse',
    'leisure',
    'man_made',
    'military',
    'natural',
    'office',
    'place',
    'power',
    'public_transport',
    'shop',
    'sport',
    'tourism',
    'water',
    'waterway',
    'wetland',
    'abandoned:aeroway',
    'abandoned:amenity',
    'abandoned:building',
    'abandoned:landuse',
    'abandoned:power',
    'area:highway'
}

-- Objects without any of the following keys will be deleted
local generic_keys = {
    'access',
    'addr:housename',
    'addr:housenumber',
    'addr:interpolation',
    'admin_level',
    'aerialway',
    'aeroway',
    'amenity',
    'area',
    'barrier',
    'bicycle',
    'boundary',
    'brand',
    'bridge',
    'building',
    'capital',
    'construction',
    'covered',
    'culvert',
    'cutting',
    'denomination',
    'disused',
    'ele',
    'embankment',
    'foot',
    'generation:source',
    'harbour',
    'healthcare',
    'highway',
    'historic',
    'hours',
    'intermittent',
    'junction',
    'landuse',
    'layer',
    'leisure',
    'lock',
    'man_made',
    'military',
    'motorcar',
    'name',
    'natural',
    'office',
    'oneway',
    'operator',
    'place',
    'population',
    'power',
    'power_source',
    'public_transport',
    'railway',
    'ref',
    'religion',
    'route',
    'service',
    'shop',
    'sport',
    'surface',
    'toll',
    'tourism',
    'tower:type',
    'tracktype',
    'tunnel',
    'water',
    'waterway',
    'wetland',
    'width',
    'wood',
    'abandoned:aeroway',
    'abandoned:amenity',
    'abandoned:building',
    'abandoned:landuse',
    'abandoned:power',
    'area:highway'
}

-- The following keys will be deleted
local delete_keys = {
    'attribution',
    'comment',
    'created_by',
    'fixme',
    'note',
    'note:*',
    'odbl',
    'odbl:note',
    'source',
    'source:*',
    'source_ref',
    'way',
    'way_area',
    'z_order',
}

local point_columns = {
    'access',
    'addr:housename',
    'addr:housenumber',
    'addr:interpolation',
    'admin_level',
    'aerialway',
    'aeroway',
    'amenity',
    'area',
    'barrier',
    'bicycle',
    'brand',
    'bridge',
    'boundary',
    'building',
    'capital',
    'construction',
    'covered',
    'culvert',
    'cutting',
    'denomination',
    'disused',
    'ele',
    'embankment',
    'foot',
    'generator:source',
    'harbour',
    'highway',
    'historic',
    'horse',
    'intermittent',
    'junction',
    'landuse',
    'layer',
    'leisure',
    'lock',
    'man_made',
    'military',
    'motorcar',
    'name',
    'name:fi',
    'name:sv',
    'minority_name',
    'majority_name',
    'bilingual_name',
    'natural',
    'office',
    'oneway',
    'operator',
    'place',
    'population',
    'power',
    'power_source',
    'public_transport',
    'railway',
    'ref',
    'religion',
    'route',
    'service',
    'shop',
    'sport',
    'surface',
    'toll',
    'tourism',
    'tower:type',
    'tunnel',
    'water',
    'waterway',
    'wetland',
    'width',
    'wood',
}

local non_point_columns = {
    'access',
    'addr:housename',
    'addr:housenumber',
    'addr:interpolation',
    'admin_level',
    'aerialway',
    'aeroway',
    'amenity',
    'area',
    'barrier',
    'bicycle',
    'brand',
    'bridge',
    'boundary',
    'building',
    'construction',
    'covered',
    'culvert',
    'cutting',
    'denomination',
    'disused',
    'embankment',
    'foot',
    'generator:source',
    'harbour',
    'highway',
    'historic',
    'horse',
    'intermittent',
    'junction',
    'landuse',
    'layer',
    'leisure',
    'lock',
    'man_made',
    'military',
    'motorcar',
    'name',
    'name:fi',
    'name:sv',
    'minority_name',
    'majority_name',
    'bilingual_name',
    'natural',
    'office',
    'oneway',
    'operator',
    'place',
    'population',
    'power',
    'power_source',
    'public_transport',
    'railway',
    'ref',
    'ref_class',
    'ref_width',
    'religion',
    'route',
    'service',
    'shop',
    'sport',
    'surface',
    'toll',
    'tourism',
    'tower:type',
    'tracktype',
    'tunnel',
    'water',
    'waterway',
    'wetland',
    'width',
    'wood',
}

-- highlight some suburbs in the capital region
boroughs = {
    -- Helsinki
    ["Q2116584"] = 1, -- Vuosaari / Nordsjö
    ["Q2510635"] = 1, -- Oulunkylä / Åggelby
    ["Q3130945"] = 1, -- Viikki / Vik
    ["Q1614778"] = 1, -- Herttoniemi / Hertonäs
    -- Espoo
    ["Q166942"] = 1, -- Leppävaara / Alberga
    ["Q3107346"] = 1, -- Matinkylä / Mattby
    ["Q211491"] = 1, -- Espoonlahti / Esboviken
    ["Q211489"] = 1, -- Espoon keskus / Esbo centrum
    ["Q219044"] = 1, -- Kauklahti / Köklax
    ["Q1668730"] = 1, -- Tapiola / Hagalund
    -- Vantaa
    ["Q2640455"] = 1, -- Tikkurila / Dickursby
    ["Q4412122"] = 1, -- Aviapolis
    ["Q3736737"] = 1, -- Myyrmäki / Myrbacka
    ["Q4557316"] = 1, -- Kivistö
    ["Q4556551"] = 1, -- Koivukylä / Björkby
    ["Q3742144"] = 1, -- Korso
    ["Q4556262"] = 1, -- Hakunila / Håkansböle
}

--
-- Name in majority language for object. This is name unless both name:fi and
-- name:sv disagree with name.
-- @param tags Raw OSM tags
-- @return Bilingual name
--
function majority_name(tags)
    if tags.name ~= nil and tags['name:sv'] ~= nil and tags['name:fi'] and tags.name ~= tags['name:fi'] and tags.name ~= tags['name:sv']
    then
        return tags['name:fi']
    else
        return tags.name
    end
end
--
-- Name in minority language for object. This is the Swedish name if name is in
-- Finnish, and the Finnish name if name is in Swedish.
-- @param tags Raw OSM tags
-- @return Bilingual name
--
function minority_name(tags)
    if tags['name:sv'] ~= nil and tags['name'] ~= tags['name:sv']
    then
        return tags['name:sv']
    elseif tags['name:fi'] ~= nil and tags['name'] ~= tags['name:fi']
    then
        return tags['name:fi']
    else
        return nil
    end
end

function bilingual_name(tags)
    if tags.majority_name ~= nil and tags.minority_name ~= nil
    then
        return tags.majority_name .. '\n' .. tags.minority_name
    else
        return tags.majority_name
    end
end

-- Makes highway classes more sensible
function highway_class(tags)
    if string.match(tags.highway, '_link$')
    then
        if tags.tags.functional_class == 'service' or tags.tags.functional_class == 'unclassified'
        then
            return tags.tags.functional_class
        elseif tags.tags.functional_class ~= nil
        then
            return tags.tags.functional_class..'_link'
        elseif tags.highway == 'motorway_link'
        then
            return 'trunk_link'
        else
            return tags.highway
        end
    elseif tags.highway == 'motorway'
    then
        if tags.tags.functional_class == 'primary' or tags.tags.functional_class == 'secondary' or tags.tags.functional_class == 'tertiary'
        then
            return tags.tags.functional_class
        elseif tags.tags.functional_class == 'service'
        then
            return 'service'
        else
            return 'trunk'
        end
    else
        if tags.highway ~= nil and tags.tags.functional_class ~= nil
        then
            return tags.tags.functional_class
        else
            return tags.highway
        end
    end
end

function has_shields(tags)
    return (tags.highway == 'trunk' or tags.highway == 'primary' or tags.highway == 'secondary' or tags.highway == 'tertiary') 
end

-- https://stackoverflow.com/a/7615129
function mysplit(inputstr, sep)
        if sep == nil then
                sep = "%s"
        end
        local t={}
        for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
                table.insert(t, str)
        end
        return t
end

function combine_stringlists(a, b)
    if a ~= nil and a ~= ''
    then
        if b ~= nil and b ~= ''
        then
            return a..';'..b
        else
            return a
        end
    else
        return b
    end
end

function combine_refs(a, b)
    return combine_stringlists(a, b)
end

-- Widths of ref characters in Liikenne font, used to calculate shield sizes
character_widths = {
    ['0'] = 616,
    ['1'] = 337,
    ['2'] = 600,
    ['3'] = 621,
    ['4'] = 689,
    ['5'] = 623,
    ['6'] = 566,
    ['7'] = 499,
    ['8'] = 616,
    ['9'] = 566,
    [' '] = 260,
    ['E'] = 613,
}

function ref_width(ref)
    local result = 0
    for i = 1, #ref
    do
        result = result + (character_widths[ref:sub(i, i)] or 0)
    end
    return math.max(math.ceil((result - 1000) / 77), 1)
end

function ref_widths(refs)
    local result = ''
    for _, ref in ipairs(mysplit(refs, ';'))
    do
        result = combine_stringlists(result, ref_width(ref))
    end
    return result
end

-- Makes refs more sensible.
-- 5-digit refs are not found on the ground so they are omitted.
-- int_ref is added to refs.
function highway_refs(tags)
    if has_shields(tags) and (tags.ref ~= nil or tags.tags.int_ref ~= nil)
    then
        if tags.junction == 'roundabout'
        then
            return nil
        else
            local result = ''
            local all_refs = combine_refs(tags.ref, tags.tags.int_ref)
            for _, ref in ipairs(mysplit(all_refs, ';'))
            do
                if tonumber(ref) == nil or tonumber(ref) < 10000
                then
                    ref = string.gsub(ref, '^E 0', 'E ')
                    result = combine_refs(result, ref)
                end
            end
            return result
        end
    else
        return tags.ref
    end
end

-- classify road type by number
function road_class_by_ref(ref)
    local n = tonumber(ref)
    if n ~= nil and n > 0
    then
        if n < 40
        then
            return 'valtatie'
        elseif n < 100
        then
            return 'kantatie'
        elseif n < 1000
        then
            return 'seututie'
        else
            return 'yhdystie'
        end
    elseif string.match(ref, '^E ?%d+$')
    then
        return 'eurooppatie'
    else
        return 'unknown'
    end
end

function classify_road_numbers(refs)
    local result = ''
    for _, ref in ipairs(mysplit(refs, ';'))
    do
        result = combine_stringlists(result, road_class_by_ref(ref))
    end
    return result
end

function gen_columns(text_columns, with_hstore, area, geometry_type)
    columns = {}

    local add_column = function (name, type)
        columns[#columns + 1] = { column = name, type = type }
    end

    for _, c in ipairs(text_columns) do
        add_column(c, 'text')
    end

    add_column('z_order', 'int')

    if area ~= nil then
        if area then
            add_column('way_area', 'area')
        else
            add_column('way_area', 'real')
        end
    end

    if hstore_column then
        add_column(hstore_column, 'hstore')
    end

    if with_hstore then
        add_column('tags', 'hstore')
    end

    add_column('way', geometry_type)

    return columns
end

local tables = {}

tables.point = osm2pgsql.define_table{
    name = prefix .. '_point',
    ids = { type = 'node', id_column = 'osm_id' },
    columns = gen_columns(point_columns, hstore or hstore_all, nil, 'point')
}

tables.line = osm2pgsql.define_table{
    name = prefix .. '_line',
    ids = { type = 'way', id_column = 'osm_id' },
    columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring')
}

tables.polygon = osm2pgsql.define_table{
    name = prefix .. '_polygon',
    ids = { type = 'area', id_column = 'osm_id' },
    columns = gen_columns(non_point_columns, hstore or hstore_all, true, 'geometry')
}

tables.roads = osm2pgsql.define_table{
    name = prefix .. '_roads',
    ids = { type = 'way', id_column = 'osm_id' },
    columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring')
}

local z_order_lookup = {
    proposed = {1, false},
    construction = {2, false},
    steps = {10, false},
    cycleway = {10, false},
    bridleway = {10, false},
    footway = {10, false},
    path = {10, false},
    track = {11, false},
    service = {15, false},

    tertiary_link = {24, false},
    secondary_link = {25, true},
    primary_link = {27, true},
    trunk_link = {28, true},
    motorway_link = {29, true},

    raceway = {30, false},
    pedestrian = {31, false},
    living_street = {32, false},
    road = {33, false},
    unclassified = {33, false},
    residential = {33, false},
    tertiary = {34, false},
    secondary = {36, true},
    primary = {37, true},
    trunk = {38, true},
    motorway = {39, true}
}

function as_bool(value)
    return value == 'yes' or value == 'true' or value == '1'
end

function get_z_order(tags)
    local z_order = 100 * math.floor(tonumber(tags.layer or '0') or 0)
    local roads = false

    local highway = tags['highway']
    if highway then
        local r = z_order_lookup[highway] or {0, false}
        z_order = z_order + r[1]
        roads = r[2]
    end

    if tags.railway then
        z_order = z_order + 35
        roads = true
    end

    if tags.boundary and tags.boundary == 'administrative' then
        roads = true
    end

    if as_bool(tags.bridge) then
        z_order = z_order + 100
    end

    if as_bool(tags.tunnel) then
        z_order = z_order - 100
    end

    return z_order, roads
end

function make_check_in_list_func(list)
    local h = {}
    for _, k in ipairs(list) do
        h[k] = true
    end
    return function(tags)
        for k, _ in pairs(tags) do
            if h[k] then
                return true
            end
        end
        return false
    end
end

local is_polygon = make_check_in_list_func(polygon_keys)
local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys)

function make_column_hash(columns)
    local h = {}

    for _, k in ipairs(columns) do
        h[k] = true
    end

    return h
end

function make_get_output(columns, hstore_all)
    local h = make_column_hash(columns)
    if hstore_all then
        return function(tags)
            local output = {}
            local hstore_entries = {}

            for k, _ in pairs(tags) do
                if h[k] then
                    output[k] = tags[k]
                end
                hstore_entries[k] = tags[k]
            end

            return output, hstore_entries
        end
    else
        return function(tags)
            local output = {}
            local hstore_entries = {}

            for k, _ in pairs(tags) do
                if h[k] then
                    output[k] = tags[k]
                else
                    hstore_entries[k] = tags[k]
                end
            end

            return output, hstore_entries
        end
    end
end

local has_generic_tag = make_check_in_list_func(generic_keys)

local get_point_output = make_get_output(point_columns, hstore_all)
local get_non_point_output = make_get_output(non_point_columns, hstore_all)

function get_hstore_column(tags)
    local len = #hstore_column
    local h = {}
    for k, v in pairs(tags) do
        if k:sub(1, len) == hstore_column then
            h[k:sub(len + 1)] = v
        end
    end

    if next(h) then
        return h
    end
    return nil
end

function add_generic_tags(object, output)
    output.majority_name = majority_name(output)
    output.minority_name = minority_name(output)
    output.bilingual_name = bilingual_name(output)
    return output
end

function osm2pgsql.process_node(object)
    if clean_tags(object.tags) then
        return
    end

    if object.tags.layer then
        object.tags.layer = tonumber(object.tags.layer)
    end

    local output
    local output_hstore = {}
    if hstore or hstore_all then
        output, output_hstore = get_point_output(object.tags)
        if not next(output) and not next(output_hstore) then
            return
        end
        if hstore_match_only and not has_generic_tag(object.tags) then
            return
        end
    else
        output = object.tags
        if not has_generic_tag(object.tags) then
            return
        end
    end

    output = add_generic_tags(object, output)
    output.tags = output_hstore

    if hstore_column then
        output[hstore_column] = get_hstore_column(object.tags)
    end

    tables.point:add_row(output)
end

function osm2pgsql.process_way(object)
    if clean_tags(object.tags) then
        return
    end
 
    if object.tags.layer then
        object.tags.layer = tonumber(object.tags.layer)
    end


    local add_area = false
    if object.tags.natural == 'coastline' then
        add_area = true
        if not keep_coastlines then
            object.tags.natural = nil
        end
    end

    local output
    local output_hstore = {}
    if hstore or hstore_all then
        output, output_hstore = get_non_point_output(object.tags)
        if not next(output) and not next(output_hstore) then
            return
        end
        if hstore_match_only and not has_generic_tag(object.tags) then
            return
        end
        if add_area and hstore_all then
            output_hstore.area = 'yes'
        end
    else
        output = object.tags
        if not has_generic_tag(object.tags) then
            return
        end
    end

    local polygon
    local area_tag = object.tags.area
    if area_tag == 'yes' or area_tag == '1' or area_tag == 'true' then
        polygon = true
    elseif area_tag == 'no' or area_tag == '0' or area_tag == 'false' then
        polygon = false
    else
        polygon = is_polygon(object.tags)
    end

    if add_area then
        output.area = 'yes'
        polygon = true
    end

    local z_order, roads = get_z_order(object.tags)
    output.z_order = z_order
    output = add_generic_tags(object, output)

    output.tags = output_hstore

    if output.highway ~= nil
    then
        if output.highway == 'construction'
        then
            return
        end
        output.highway = highway_class(output)
        output.ref = highway_refs(output)
        if output.highway == 'construction' and output.construction == 'motorway'
        then
            output.construction = 'trunk'
        elseif output.highway == 'construction' and output.construction == 'motorway_link'
        then
            output.construction = 'trunk_link'
        end

        -- classify references for for Finland shield rendering
        if output.ref ~= nil
        then
            output.ref_class = classify_road_numbers(output.ref)
            output.ref_width = ref_widths(output.ref)
        end
    end

    if hstore_column then
        output[hstore_column] = get_hstore_column(object.tags)
    end

    if polygon and object.is_closed then
        output.way = { create = 'area' }
        tables.polygon:add_row(output)
    else
        output.way = { create = 'line', split_at = max_length }
        tables.line:add_row(output)
        if roads then
            tables.roads:add_row(output)
        end
    end
end

function osm2pgsql.process_relation(object)
    if clean_tags(object.tags) then
        return
    end

    local type = object.tags.type
    if (type ~= 'route') and (type ~= 'multipolygon') and (type ~= 'boundary') then
        return
    end
    object.tags.type = nil

    local output
    local output_hstore = {}
    if hstore or hstore_all then
        output, output_hstore = get_non_point_output(object.tags)
        if not next(output) and not next(output_hstore) then
            return
        end
        if hstore_match_only and not has_generic_tag(object.tags) then
            return
        end
    else
        output = object.tags
        if not has_generic_tag(object.tags) then
            return
        end
    end

    if not next(output) and not next(output_hstore) then
        return
    end

    if enable_legacy_route_processing and (hstore or hstore_all) and type == 'route' then
        if not object.tags.route_name then
            output_hstore.route_name = object.tags.name
        end

        local state = object.tags.state
        if state ~= 'alternate' and state ~= 'connection' then
            state = 'yes'
        end

        local network = object.tags.network
        if network == 'lcn' then
            output_hstore.lcn = output_hstore.lcn or state
            output_hstore.lcn_ref = output_hstore.lcn_ref or object.tags.ref
        elseif network == 'rcn' then
            output_hstore.rcn = output_hstore.rcn or state
            output_hstore.rcn_ref = output_hstore.rcn_ref or object.tags.ref
        elseif network == 'ncn' then
            output_hstore.ncn = output_hstore.ncn or state
            output_hstore.ncn_ref = output_hstore.ncn_ref or object.tags.ref
        elseif network == 'lwn' then
            output_hstore.lwn = output_hstore.lwn or state
            output_hstore.lwn_ref = output_hstore.lwn_ref or object.tags.ref
        elseif network == 'rwn' then
            output_hstore.rwn = output_hstore.rwn or state
            output_hstore.rwn_ref = output_hstore.rwn_ref or object.tags.ref
        elseif network == 'nwn' then
            output_hstore.nwn = output_hstore.nwn or state
            output_hstore.nwn_ref = output_hstore.nwn_ref or object.tags.ref
        end

        local pc = object.tags.preferred_color
        if pc == '0' or pc == '1' or pc == '2' or pc == '3' or pc == '4' then
            output_hstore.route_pref_color = pc
        else
            output_hstore.route_pref_color = '0'
        end
    end

    local make_boundary = false
    local make_polygon = false
    if type == 'boundary' then
        make_boundary = true
    elseif type == 'multipolygon' and object.tags.boundary then
        make_boundary = true
    elseif type == 'multipolygon' then
        make_polygon = true
    end

    local z_order, roads = get_z_order(object.tags)
    output.z_order = z_order
    output = add_generic_tags(object, output)

    output.tags = output_hstore

    if hstore_column then
        output[hstore_column] = get_hstore_column(object.tags)
    end

    if not make_polygon then
        output.way = { create = 'line', split_at = max_length }
        tables.line:add_row(output)
        if roads then
            tables.roads:add_row(output)
        end
    end

    if make_boundary or make_polygon then
        output.way = { create = 'area', multi = multi_geometry }
        tables.polygon:add_row(output)
    end
end