tip/flex.lua

-- This configuration for the flex backend tries to be compatible with the
-- original pgsql c-transform backend. There might be some corner cases but
-- it should mostly do exactly the same.

-- Set this to true if you were using option -K|--keep-coastlines.
local keep_coastlines = true

-- Set this to the table name prefix (what used to be option -p|--prefix).
local prefix = 'planet_osm'

-- Set this to true if multipolygons should be written as polygons into db
-- (what used to be option -G|--multi-geometry).
local multi_geometry = true

-- Set this to true if you want an hstore column (what used to be option
-- -k|--hstore). Can not be true if "hstore_all" is true.
local hstore = true

-- Set this to true if you want all tags in an hstore column (what used to
-- be option -j|--hstore-all). Can not be true if "hstore" is true.
local hstore_all = false

-- Only keep objects that have a value in one of the non-hstore columns
-- (normal action with --hstore is to keep all objects). Equivalent to
-- what used to be set through option --hstore-match-only.
local hstore_match_only = true

-- Set this to add an additional hstore (key/value) column containing all tags
-- that start with the specified string, eg "name:". Will produce an extra
-- hstore column that contains all "name:xx" tags. Equivalent to what used to
-- be set through option -z|--hstore-column.
local hstore_column = nil

-- There is some very old specialized handling of route relations in osm2pgsql,
-- which you probably don't need. This is disabled here, but you can enable
-- it by setting this to true. If you don't understand this, leave it alone.
local enable_legacy_route_processing = false

-- ---------------------------------------------------------------------------

if hstore and hstore_all then
	error("hstore and hstore_all can't be both true")
end

-- Used for splitting up long linestrings
if osm2pgsql.srid == 4326 then
	max_length = 1
else
	max_length = 100000
end

-- Ways with any of the following keys will be treated as polygon
local polygon_keys = {
	'aeroway',
	'amenity',
	'building',
	'harbour',
	'historic',
	'landuse',
	'leisure',
	'man_made',
	'military',
	'natural',
	'office',
	'place',
	'power',
	'public_transport',
	'shop',
	'sport',
	'tourism',
	'water',
	'waterway',
	'wetland',
	'abandoned:aeroway',
	'abandoned:amenity',
	'abandoned:building',
	'abandoned:landuse',
	'abandoned:power',
	'area:highway'
}

-- Objects without any of the following keys will be deleted
local generic_keys = {
	'access',
	'addr:housename',
	'addr:housenumber',
	'addr:interpolation',
	'admin_level',
	'aerialway',
	'aeroway',
	'amenity',
	'area',
	'barrier',
	'bicycle',
	'boundary',
	'brand',
	'bridge',
	'building',
	'capital',
	'construction',
	'covered',
	'culvert',
	'cutting',
	'denomination',
	'disused',
	'ele',
	'embankment',
	'foot',
	'generation:source',
	'harbour',
	'healthcare',
	'highway',
	'historic',
	'hours',
	'intermittent',
	'junction',
	'landuse',
	'layer',
	'leisure',
	'lock',
	'man_made',
	'military',
	'motorcar',
	'name',
	'natural',
	'office',
	'oneway',
	'operator',
	'place',
	'population',
	'power',
	'power_source',
	'public_transport',
	'railway',
	'ref',
	'religion',
	'route',
	'service',
	'shop',
	'sport',
	'surface',
	'toll',
	'tourism',
	'tower:type',
	'tracktype',
	'tunnel',
	'water',
	'waterway',
	'wetland',
	'width',
	'wood',
	'abandoned:aeroway',
	'abandoned:amenity',
	'abandoned:building',
	'abandoned:landuse',
	'abandoned:power',
	'area:highway'
}

-- The following keys will be deleted
local delete_keys = {
	'attribution',
	'comment',
	'created_by',
	'fixme',
	'note',
	'note:*',
	'odbl',
	'odbl:note',
	'source',
	'source:*',
	'source_ref',
	'way',
	'way_area',
	'z_order',
}

local point_columns = {
	'access',
	'addr:housename',
	'addr:housenumber',
	'addr:interpolation',
	'admin_level',
	'aerialway',
	'aeroway',
	'amenity',
	'area',
	'barrier',
	'bicycle',
	'brand',
	'bridge',
	'boundary',
	'building',
	'capital',
	'construction',
	'covered',
	'culvert',
	'cutting',
	'denomination',
	'disused',
	'ele',
	'embankment',
	'foot',
	'generator:source',
	'harbour',
	'highway',
	'historic',
	'horse',
	'intermittent',
	'junction',
	'landuse',
	'layer',
	'leisure',
	'lock',
	'man_made',
	'military',
	'motorcar',
	'name',
	'name:fi',
	'name:sv',
	'minority_name',
	'majority_name',
	'bilingual_name',
	'natural',
	'office',
	'oneway',
	'operator',
	'place',
	'population',
	'power',
	'power_source',
	'public_transport',
	'railway',
	'ref',
	'religion',
	'route',
	'service',
	'shop',
	'sport',
	'surface',
	'toll',
	'tourism',
	'tower:type',
	'tunnel',
	'water',
	'waterway',
	'wetland',
	'width',
	'wood',
}

local non_point_columns = {
	'access',
	'addr:housename',
	'addr:housenumber',
	'addr:interpolation',
	'admin_level',
	'aerialway',
	'aeroway',
	'amenity',
	'area',
	'barrier',
	'bicycle',
	'brand',
	'bridge',
	'boundary',
	'building',
	'construction',
	'covered',
	'culvert',
	'cutting',
	'denomination',
	'disused',
	'embankment',
	'foot',
	'generator:source',
	'harbour',
	'highway',
	'historic',
	'horse',
	'intermittent',
	'junction',
	'landuse',
	'layer',
	'leisure',
	'lock',
	'man_made',
	'military',
	'motorcar',
	'name',
	'name:fi',
	'name:sv',
	'minority_name',
	'majority_name',
	'bilingual_name',
	'natural',
	'office',
	'oneway',
	'operator',
	'place',
	'population',
	'power',
	'power_source',
	'public_transport',
	'railway',
	'ref',
	'ref_class',
	'ref_width',
	'religion',
	'route',
	'service',
	'shop',
	'sport',
	'surface',
	'toll',
	'tourism',
	'tower:type',
	'tracktype',
	'tunnel',
	'water',
	'waterway',
	'wetland',
	'width',
	'wood',
}

-- highlight some suburbs in the capital region
boroughs = {
	-- Helsinki
	["Q2116584"] = 1, -- Vuosaari / Nordsjö
	["Q2510635"] = 1, -- Oulunkylä / Åggelby
	["Q3130945"] = 1, -- Viikki / Vik
	["Q1614778"] = 1, -- Herttoniemi / Hertonäs
	-- Espoo
	["Q166942"] = 1, -- Leppävaara / Alberga
	["Q3107346"] = 1, -- Matinkylä / Mattby
	["Q211491"] = 1, -- Espoonlahti / Esboviken
	["Q211489"] = 1, -- Espoon keskus / Esbo centrum
	["Q219044"] = 1, -- Kauklahti / Köklax
	["Q1668730"] = 1, -- Tapiola / Hagalund
	-- Vantaa
	["Q2640455"] = 1, -- Tikkurila / Dickursby
	["Q4412122"] = 1, -- Aviapolis
	["Q3736737"] = 1, -- Myyrmäki / Myrbacka
	["Q4557316"] = 1, -- Kivistö
	["Q4556551"] = 1, -- Koivukylä / Björkby
	["Q3742144"] = 1, -- Korso
	["Q4556262"] = 1, -- Hakunila / Håkansböle
}

--
-- Name in majority language for object. This is name unless both name:fi and
-- name:sv disagree with name.
-- @param tags Raw OSM tags
-- @return Bilingual name
--
function majority_name(tags)
	if tags.name ~= nil and tags['name:sv'] ~= nil and tags['name:fi'] and tags.name ~= tags['name:fi'] and tags.name ~= tags['name:sv']
	then
		return tags['name:fi']
	else
		return tags.name
	end
end
--
-- Name in minority language for object. This is the Swedish name if name is in
-- Finnish, and the Finnish name if name is in Swedish.
-- @param tags Raw OSM tags
-- @return Bilingual name
--
function minority_name(tags)
	if tags['name:sv'] ~= nil and tags['name'] ~= tags['name:sv']
	then
		return tags['name:sv']
	elseif tags['name:fi'] ~= nil and tags['name'] ~= tags['name:fi']
	then
		return tags['name:fi']
	else
		return nil
	end
end

function bilingual_name(tags)
	if tags.majority_name ~= nil and tags.minority_name ~= nil
	then
		return tags.majority_name .. '\n' .. tags.minority_name
	else
		return tags.majority_name
	end
end

-- Makes highway classes more sensible
function highway_class(tags)
	if string.match(tags.highway, '_link$')
	then
		if tags.tags.functional_class == 'service' or tags.tags.functional_class == 'unclassified'
		then
			return tags.tags.functional_class
		elseif tags.tags.functional_class ~= nil
		then
			return tags.tags.functional_class..'_link'
		elseif tags.highway == 'motorway_link'
		then
			return 'trunk_link'
		else
			return tags.highway
		end
	elseif tags.highway == 'motorway'
	then
		if tags.tags.functional_class == 'primary' or tags.tags.functional_class == 'secondary' or tags.tags.functional_class == 'tertiary'
		then
			return tags.tags.functional_class
		elseif tags.tags.functional_class == 'service'
		then
			return 'service'
		else
			return 'trunk'
		end
	else
		if tags.highway ~= nil and tags.tags.functional_class ~= nil
		then
			return tags.tags.functional_class
		else
			return tags.highway
		end
	end
end

function has_shields(tags)
	return (tags.highway == 'trunk' or tags.highway == 'primary' or tags.highway == 'secondary' or tags.highway == 'tertiary') 
end

-- https://stackoverflow.com/a/7615129
function mysplit(inputstr, sep)
		if sep == nil then
				sep = "%s"
		end
		local t={}
		for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
				table.insert(t, str)
		end
		return t
end

function combine_stringlists(a, b)
	if a ~= nil and a ~= ''
	then
		if b ~= nil and b ~= ''
		then
			return a..';'..b
		else
			return a
		end
	else
		return b
	end
end

function combine_refs(a, b)
	return combine_stringlists(a, b)
end

-- Widths of ref characters in Liikenne font, used to calculate shield sizes
character_widths = {
	['0'] = 616,
	['1'] = 337,
	['2'] = 600,
	['3'] = 621,
	['4'] = 689,
	['5'] = 623,
	['6'] = 566,
	['7'] = 499,
	['8'] = 616,
	['9'] = 566,
	[' '] = 260,
	['E'] = 613,
}

function ref_width(ref)
	local result = 0
	for i = 1, #ref
	do
		result = result + (character_widths[ref:sub(i, i)] or 0)
	end
	return math.max(math.ceil((result - 1000) / 77), 1)
end

function ref_widths(refs)
	local result = ''
	for _, ref in ipairs(mysplit(refs, ';'))
	do
		result = combine_stringlists(result, ref_width(ref))
	end
	return result
end

-- Makes refs more sensible.
-- 5-digit refs are not found on the ground so they are omitted.
-- int_ref is added to refs.
function highway_refs(tags)
	if has_shields(tags) and (tags.ref ~= nil or tags.tags.int_ref ~= nil)
	then
		if tags.junction == 'roundabout'
		then
			return nil
		else
			local result = ''
			local all_refs = combine_refs(tags.ref, tags.tags.int_ref)
			for _, ref in ipairs(mysplit(all_refs, ';'))
			do
				if tonumber(ref) == nil or tonumber(ref) < 10000
				then
					ref = string.gsub(ref, '^E 0', 'E ')
					result = combine_refs(result, ref)
				end
			end
			return result
		end
	else
		return tags.ref
	end
end

-- classify road type by number
function road_class_by_ref(ref)
	local n = tonumber(ref)
	if n ~= nil and n > 0
	then
		if n < 40
		then
			return 'valtatie'
		elseif n < 100
		then
			return 'kantatie'
		elseif n < 1000
		then
			return 'seututie'
		else
			return 'yhdystie'
		end
	elseif string.match(ref, '^E ?%d+$')
	then
		return 'eurooppatie'
	else
		return 'unknown'
	end
end

function classify_road_numbers(refs)
	local result = ''
	for _, ref in ipairs(mysplit(refs, ';'))
	do
		result = combine_stringlists(result, road_class_by_ref(ref))
	end
	return result
end

function gen_columns(text_columns, with_hstore, area, geometry_type)
	columns = {}

	local add_column = function (name, type)
		columns[#columns + 1] = { column = name, type = type }
	end

	for _, c in ipairs(text_columns) do
		add_column(c, 'text')
	end

	add_column('z_order', 'int')

	if area ~= nil then
		if area then
			add_column('way_area', 'area')
		else
			add_column('way_area', 'real')
		end
	end

	if hstore_column then
		add_column(hstore_column, 'hstore')
	end

	if with_hstore then
		add_column('tags', 'hstore')
	end

	add_column('way', geometry_type)

	return columns
end

local tables = {}

tables.point = osm2pgsql.define_table{
	name = prefix .. '_point',
	ids = { type = 'node', id_column = 'osm_id' },
	columns = gen_columns(point_columns, hstore or hstore_all, nil, 'point')
}

tables.line = osm2pgsql.define_table{
	name = prefix .. '_line',
	ids = { type = 'way', id_column = 'osm_id' },
	columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring')
}

tables.polygon = osm2pgsql.define_table{
	name = prefix .. '_polygon',
	ids = { type = 'area', id_column = 'osm_id' },
	columns = gen_columns(non_point_columns, hstore or hstore_all, true, 'geometry')
}

tables.roads = osm2pgsql.define_table{
	name = prefix .. '_roads',
	ids = { type = 'way', id_column = 'osm_id' },
	columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring')
}

tables.routes = osm2pgsql.define_way_table(
	prefix .. '_routes',
	{
		{ column = 'tags', type = 'hstore' },
		{ column = 'ref', type = 'text' },
		{ column = 'name', type = 'text' },
		{ column = 'network', type = 'text' },
		{ column = 'route', type = 'text' },
		{ column = 'layer', type = 'text' },
		{ column = 'osm_way_id', type = 'text' },
		{ column = 'way', type = 'linestring' },
	}
)


local z_order_lookup = {
	proposed = {1, false},
	construction = {2, false},
	steps = {10, false},
	cycleway = {10, false},
	bridleway = {10, false},
	footway = {10, false},
	path = {10, false},
	track = {11, false},
	service = {15, false},

	tertiary_link = {24, false},
	secondary_link = {25, true},
	primary_link = {27, true},
	trunk_link = {28, true},
	motorway_link = {29, true},

	raceway = {30, false},
	pedestrian = {31, false},
	living_street = {32, false},
	road = {33, false},
	unclassified = {33, false},
	residential = {33, false},
	tertiary = {34, false},
	secondary = {36, true},
	primary = {37, true},
	trunk = {38, true},
	motorway = {39, true}
}

function as_bool(value)
	return value == 'yes' or value == 'true' or value == '1'
end

function get_z_order(tags)
	local z_order = 100 * math.floor(tonumber(tags.layer or '0') or 0)
	local roads = false

	local highway = tags['highway']
	if highway then
		local r = z_order_lookup[highway] or {0, false}
		z_order = z_order + r[1]
		roads = r[2]
	end

	if tags.railway then
		z_order = z_order + 35
		roads = true
	end

	if tags.boundary and tags.boundary == 'administrative' then
		roads = true
	end

	if as_bool(tags.bridge) then
		z_order = z_order + 100
	end

	if as_bool(tags.tunnel) then
		z_order = z_order - 100
	end

	return z_order, roads
end

function make_check_in_list_func(list)
	local h = {}
	for _, k in ipairs(list) do
		h[k] = true
	end
	return function(tags)
		for k, _ in pairs(tags) do
			if h[k] then
				return true
			end
		end
		return false
	end
end

local is_polygon = make_check_in_list_func(polygon_keys)
local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys)

function make_column_hash(columns)
	local h = {}

	for _, k in ipairs(columns) do
		h[k] = true
	end

	return h
end

function make_get_output(columns, hstore_all)
	local h = make_column_hash(columns)
	if hstore_all then
		return function(tags)
			local output = {}
			local hstore_entries = {}

			for k, _ in pairs(tags) do
				if h[k] then
					output[k] = tags[k]
				end
				hstore_entries[k] = tags[k]
			end

			return output, hstore_entries
		end
	else
		return function(tags)
			local output = {}
			local hstore_entries = {}

			for k, _ in pairs(tags) do
				if h[k] then
					output[k] = tags[k]
				else
					hstore_entries[k] = tags[k]
				end
			end

			return output, hstore_entries
		end
	end
end

local has_generic_tag = make_check_in_list_func(generic_keys)

local get_point_output = make_get_output(point_columns, hstore_all)
local get_non_point_output = make_get_output(non_point_columns, hstore_all)

function get_hstore_column(tags)
	local len = #hstore_column
	local h = {}
	for k, v in pairs(tags) do
		if k:sub(1, len) == hstore_column then
			h[k:sub(len + 1)] = v
		end
	end

	if next(h) then
		return h
	end
	return nil
end

function add_generic_tags(object, output)
	output.majority_name = majority_name(output)
	output.minority_name = minority_name(output)
	output.bilingual_name = bilingual_name(output)
	return output
end

function osm2pgsql.process_node(object)
	if clean_tags(object.tags) then
		return
	end

	if object.tags.layer then
		object.tags.layer = tonumber(object.tags.layer)
	end

	local output
	local output_hstore = {}
	if hstore or hstore_all then
		output, output_hstore = get_point_output(object.tags)
		if not next(output) and not next(output_hstore) then
			return
		end
		if hstore_match_only and not has_generic_tag(object.tags) then
			return
		end
	else
		output = object.tags
		if not has_generic_tag(object.tags) then
			return
		end
	end

	output = add_generic_tags(object, output)
	output.tags = output_hstore

	if hstore_column then
		output[hstore_column] = get_hstore_column(object.tags)
	end

	tables.point:add_row(output)
end

route_ways = {}

function osm2pgsql.process_way(object)
	if clean_tags(object.tags) then
		return
	end
 
	if object.tags.layer then
		object.tags.layer = tonumber(object.tags.layer)
	end


	local add_area = false
	if object.tags.natural == 'coastline' then
		add_area = true
		if not keep_coastlines then
			object.tags.natural = nil
		end
	end

	local output
	local output_hstore = {}
	if hstore or hstore_all then
		output, output_hstore = get_non_point_output(object.tags)
		if not next(output) and not next(output_hstore) then
			return
		end
		if hstore_match_only and not has_generic_tag(object.tags) then
			return
		end
		if add_area and hstore_all then
			output_hstore.area = 'yes'
		end
	else
		output = object.tags
		if not has_generic_tag(object.tags) then
			return
		end
	end

	local polygon
	local area_tag = object.tags.area
	if area_tag == 'yes' or area_tag == '1' or area_tag == 'true' then
		polygon = true
	elseif area_tag == 'no' or area_tag == '0' or area_tag == 'false' then
		polygon = false
	else
		polygon = is_polygon(object.tags)
	end

	if add_area then
		output.area = 'yes'
		polygon = true
	end

	local z_order, roads = get_z_order(object.tags)
	output.z_order = z_order
	output = add_generic_tags(object, output)

	output.tags = output_hstore

	if output.highway ~= nil
	then
		if output.highway == 'construction'
		then
			return
		end
		output.highway = highway_class(output)
		output.ref = highway_refs(output)
		if output.highway == 'construction' and output.construction == 'motorway'
		then
			output.construction = 'trunk'
		elseif output.highway == 'construction' and output.construction == 'motorway_link'
		then
			output.construction = 'trunk_link'
		end

		-- classify references for for Finland shield rendering
		if output.ref ~= nil
		then
			output.ref_class = classify_road_numbers(output.ref)
			output.ref_width = ref_widths(output.ref)
		end
	end

	if hstore_column then
		output[hstore_column] = get_hstore_column(object.tags)
	end

	if polygon and object.is_closed then
		output.way = { create = 'area' }
		tables.polygon:add_row(output)
	else
		output.way = { create = 'line', split_at = max_length }
		tables.line:add_row(output)
		if roads then
			tables.roads:add_row(output)
		end
	end

	if osm2pgsql.stage == 2
	then
		local routes = route_ways[object.id]
		if routes ~= nil
		then
			for index, route in ipairs(routes) do
				route_type = route.tags.route
				if route_type == 'bicycle' and route.tags.network ~= nil
				then
					route_type = route_type .. '_' .. route.tags.network
				end
				row = {
					tags = object.tags,
					ref = route.tags.ref,
					name = route.tags.name,
					network = route.tags.network,
					route = route_type,
					osm_way_id = object.id,
					layer = object.tags.layer or 0,
					geom = { create = 'line' }
				}
				tables.routes:add_row(row)
			end
		end
	end
end


function osm2pgsql.select_relation_members(relation)
	if relation.tags.type == 'route' and (relation.tags.route == 'foot' or relation.tags.route == 'hiking' or relation.tags.route == 'bicycle' or relation.tags.route == 'train' or relation.tags.route == 'bus')
	then
		for _, member in ipairs(relation.members) do
			if member.type == 'w' then
				member_id = member.ref
				if not route_ways[member_id] then
					route_ways[member_id] = {}
				end
				table.insert(route_ways[member_id], relation)
			end
		end
		return { ways = osm2pgsql.way_member_ids(relation) }
	end
end


function osm2pgsql.process_relation(object)
	if clean_tags(object.tags) then
		return
	end

	local type = object.tags.type
	if (type ~= 'route') and (type ~= 'multipolygon') and (type ~= 'boundary') then
		return
	end
	object.tags.type = nil

	local output
	local output_hstore = {}
	if hstore or hstore_all then
		output, output_hstore = get_non_point_output(object.tags)
		if not next(output) and not next(output_hstore) then
			return
		end
		if hstore_match_only and not has_generic_tag(object.tags) then
			return
		end
	else
		output = object.tags
		if not has_generic_tag(object.tags) then
			return
		end
	end

	if not next(output) and not next(output_hstore) then
		return
	end

	if enable_legacy_route_processing and (hstore or hstore_all) and type == 'route' then
		if not object.tags.route_name then
			output_hstore.route_name = object.tags.name
		end

		local state = object.tags.state
		if state ~= 'alternate' and state ~= 'connection' then
			state = 'yes'
		end

		local network = object.tags.network
		if network == 'lcn' then
			output_hstore.lcn = output_hstore.lcn or state
			output_hstore.lcn_ref = output_hstore.lcn_ref or object.tags.ref
		elseif network == 'rcn' then
			output_hstore.rcn = output_hstore.rcn or state
			output_hstore.rcn_ref = output_hstore.rcn_ref or object.tags.ref
		elseif network == 'ncn' then
			output_hstore.ncn = output_hstore.ncn or state
			output_hstore.ncn_ref = output_hstore.ncn_ref or object.tags.ref
		elseif network == 'lwn' then
			output_hstore.lwn = output_hstore.lwn or state
			output_hstore.lwn_ref = output_hstore.lwn_ref or object.tags.ref
		elseif network == 'rwn' then
			output_hstore.rwn = output_hstore.rwn or state
			output_hstore.rwn_ref = output_hstore.rwn_ref or object.tags.ref
		elseif network == 'nwn' then
			output_hstore.nwn = output_hstore.nwn or state
			output_hstore.nwn_ref = output_hstore.nwn_ref or object.tags.ref
		end

		local pc = object.tags.preferred_color
		if pc == '0' or pc == '1' or pc == '2' or pc == '3' or pc == '4' then
			output_hstore.route_pref_color = pc
		else
			output_hstore.route_pref_color = '0'
		end
	end

	local make_boundary = false
	local make_polygon = false
	if type == 'boundary' then
		make_boundary = true
	elseif type == 'multipolygon' and object.tags.boundary then
		make_boundary = true
	elseif type == 'multipolygon' then
		make_polygon = true
	end

	local z_order, roads = get_z_order(object.tags)
	output.z_order = z_order
	output = add_generic_tags(object, output)

	output.tags = output_hstore

	if hstore_column then
		output[hstore_column] = get_hstore_column(object.tags)
	end

	if not make_polygon then
		output.way = { create = 'line', split_at = max_length }
		tables.line:add_row(output)
		if roads then
			tables.roads:add_row(output)
		end
	end

	if make_boundary or make_polygon then
		output.way = { create = 'area', multi = multi_geometry }
		tables.polygon:add_row(output)
	end
end