diff -r 2c78e68d7363 -r a5bfd99bc2a3 buses.py --- a/buses.py Thu Jun 22 19:01:31 2017 +0300 +++ b/buses.py Sat Jun 24 19:38:05 2017 +0300 @@ -154,294 +154,314 @@ services = {} bus_stops = {} all_clusters = set() - -print('Ladataan routes... ', file = stderr, end = '', flush = True) -with open('gtfs/routes.txt') as file: - for row in read_csv(file): - route = BusRoute(row) - routes[route.reference] = route - routes_per_id[route.id] = route -print('%d linjaa' % len(routes), file = stderr) - -print('Ladataan ajovuorot... ', file = stderr, end = '', flush = True) - -shape_distances = {} -with open('gtfs/shapes.txt') as file: - for row in read_csv(file): - shape_distances[row['shape_id']] = max(shape_distances.get(row['shape_id'], 0), float(row['shape_dist_traveled'])) - -with open('gtfs/trips.txt') as file: - for row in read_csv(file): - if row['service_id'] not in services: - services[row['service_id']] = BusService(row['service_id']) - route = routes_per_id[row['route_id']] - trip = BusTrip( - reference = row['trip_id'], - route = route, - service = services[row['service_id']], - length = shape_distances[row['shape_id']] - ) - route.trips.add(trip) - assert trip.name not in all_trips - all_trips[trip.name] = trip -print('%d ajoa' % len(all_trips), file = stderr) - -def read_date(teksti): - return date(int(teksti[:4]), int(teksti[4:6]), int(teksti[6:])) - -def read_time(teksti): - tunti, minuutti, sekunti = map(int, teksti.split(':')) - return timedelta(hours = tunti, minutes = minuutti, seconds = sekunti) - -print('Ladataan päiväykset... ', file = stderr, flush = True) - -viimeinen_käyttöpäivä = date.today() +viimeinen_käyttöpäivä = None +clusters_by_name = {} services_for_day = {} -def date_range(start_date, end_date, *, include_end = False): - ''' Generates date from start_date to end_date. If include_end is True, then end_date will be yielded. ''' - current_date = start_date - while current_date < end_date: - yield current_date - current_date += timedelta(1) - if include_end: - yield end_date +def load_buses(gtfs_zip_path, profile): + global viimeinen_käyttöpäivä + from zipfile import ZipFile + with ZipFile(gtfs_zip_path) as gtfs_zip: + print('Ladataan linjat... ', file = stderr, end = '', flush = True) + with gtfs_zip.open('routes.txt') as file: + for row in read_csv(map(bytes.decode, file)): + route = BusRoute(row) + routes[route.reference] = route + routes_per_id[route.id] = route + print('%d linjaa' % len(routes), file = stderr) + + print('Ladataan ajovuorot... ', file = stderr, end = '', flush = True) + + shape_distances = {} + with gtfs_zip.open('shapes.txt') as file: + for row in read_csv(map(bytes.decode, file)): + shape_distances[row['shape_id']] = max(shape_distances.get(row['shape_id'], 0), float(row['shape_dist_traveled'])) -def add_day_to_service(service_name, day): - try: - service = services[service_name] - except KeyError: - return - else: - service.dates.add(day) - if day not in services_for_day: - services_for_day[day] = set() - services_for_day[day].add(service) - global viimeinen_käyttöpäivä - viimeinen_käyttöpäivä = max(day, viimeinen_käyttöpäivä) + with gtfs_zip.open('trips.txt') as file: + for row in read_csv(map(bytes.decode, file)): + if row['service_id'] not in services: + services[row['service_id']] = BusService(row['service_id']) + route = routes_per_id[row['route_id']] + trip = BusTrip( + reference = row['trip_id'], + route = route, + service = services[row['service_id']], + length = shape_distances[row['shape_id']] * float(profile['metrics']['shape-modifier']) + ) + route.trips.add(trip) + assert trip.name not in all_trips + all_trips[trip.name] = trip + print('%d ajoa' % len(all_trips), file = stderr) -def filter_day(row, day): - day_names = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] - return int(row[day_names[day.isoweekday() - 1]]) + def read_date(teksti): + return date(int(teksti[:4]), int(teksti[4:6]), int(teksti[6:])) + + def read_time(teksti): + tunti, minuutti, sekunti = map(int, teksti.split(':')) + return timedelta(hours = tunti, minutes = minuutti, seconds = sekunti) + + print('Ladataan päiväykset... ', file = stderr, flush = True) + + viimeinen_käyttöpäivä = date.today() -with open('gtfs/calendar.txt') as file: - for row in read_csv(file): - for day in date_range(read_date(row['start_date']), read_date(row['end_date']), include_end = True): - if filter_day(row, day): - add_day_to_service(service_name = row['service_id'], day = day) + def date_range(start_date, end_date, *, include_end = False): + ''' Generates date from start_date to end_date. If include_end is True, then end_date will be yielded. ''' + current_date = start_date + while current_date < end_date: + yield current_date + current_date += timedelta(1) + if include_end: + yield end_date -with open('gtfs/calendar_dates.txt') as file: - for row in read_csv(file): - add_day_to_service(service_name = row['service_id'], day = read_date(row['date'])) + def add_day_to_service(service_name, day): + try: + service = services[service_name] + except KeyError: + return + else: + service.dates.add(day) + if day not in services_for_day: + services_for_day[day] = set() + services_for_day[day].add(service) + global viimeinen_käyttöpäivä + viimeinen_käyttöpäivä = max(day, viimeinen_käyttöpäivä) -def services_available_at(day): - for service in services.values(): - if day in service.dates: - yield service + def filter_day(row, day): + day_names = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + return int(row[day_names[day.isoweekday() - 1]]) -print('Ladataan pysäkit... ', file = stderr, end = '', flush = True) -with open('gtfs/stops.txt') as file: - for row in read_csv(file): - location = Sijainti(float(row['stop_lat']), float(row['stop_lon'])) - stop = BusStop( - reference = row['stop_id'], - name = row['stop_name'], - location = location, - code = row['stop_code'], - ) - bus_stops[stop.reference] = stop -with open('regions-per-stop.json') as file: - for stop_reference, region in json.load(file).items(): - bus_stops[stop_reference].region = region -print('%d pysäkkiä' % len(bus_stops), file = stderr) + with gtfs_zip.open('calendar.txt') as file: + for row in read_csv(map(bytes.decode, file)): + for day in date_range(read_date(row['start_date']), read_date(row['end_date']), include_end = True): + if filter_day(row, day): + add_day_to_service(service_name = row['service_id'], day = day) + + with gtfs_zip.open('calendar_dates.txt') as file: + for row in read_csv(map(bytes.decode, file)): + add_day_to_service(service_name = row['service_id'], day = read_date(row['date'])) + + def services_available_at(day): + for service in services.values(): + if day in service.dates: + yield service + + print('Ladataan pysäkit... ', file = stderr, end = '', flush = True) + with gtfs_zip.open('stops.txt') as file: + for row in read_csv(map(bytes.decode, file)): + location = Sijainti(float(row['stop_lat']), float(row['stop_lon'])) + stop = BusStop( + reference = row['stop_id'], + name = row['stop_name'], + location = location, + code = row['stop_code'], + ) + bus_stops[stop.reference] = stop + with open('regions-per-stop.json') as file: + for stop_reference, region in json.load(file).items(): + bus_stops[stop_reference].region = region + print('%d pysäkkiä' % len(bus_stops), file = stderr) -class BusStopCluster: - def __init__(self): - self.stops = set() - self._center = None - self.name = None - @property - def url_name(self): - return self.name.lower().replace('(', '').replace(')', '').replace(' ', '-') - def add_stop(self, stop): - assert not stop.cluster - stop.cluster = self - self.stops.add(stop) - self._center = None - @property - def center(self): - if not self._center: - if self.stops: - from statistics import median - pointtype = type(next(iter(self.stops)).location) - self._center = pointtype( - median(stop.location.x for stop in self.stops), - median(stop.location.y for stop in self.stops), - ) - else: - raise ValueError('an empty cluster has no center point') - return self._center - def merge(self, other): - for bus_stop in other.stops: - bus_stop.cluster = self - self.stops |= other.stops - other.stops = set() - other._center = None - def schedule(self, max_amount = 50): - result = [] - for stop in self.stops: - result += stop.schedule(max_amount) - result.sort(key = lambda schedule_entry: schedule_entry['time']) - return result[:max_amount] + class BusStopCluster: + def __init__(self): + self.stops = set() + self._center = None + self.name = None + @property + def url_name(self): + return self.name.lower().replace('(', '').replace(')', '').replace(' ', '-') + def add_stop(self, stop): + assert not stop.cluster + stop.cluster = self + self.stops.add(stop) + self._center = None + @property + def center(self): + if not self._center: + if self.stops: + from statistics import median + pointtype = type(next(iter(self.stops)).location) + self._center = pointtype( + median(stop.location.x for stop in self.stops), + median(stop.location.y for stop in self.stops), + ) + else: + raise ValueError('an empty cluster has no center point') + return self._center + def merge(self, other): + for bus_stop in other.stops: + bus_stop.cluster = self + self.stops |= other.stops + other.stops = set() + other._center = None + def schedule(self, max_amount = 50): + result = [] + for stop in self.stops: + result += stop.schedule(max_amount) + result.sort(key = lambda schedule_entry: schedule_entry['time']) + return result[:max_amount] -from collections import defaultdict -bus_stops_by_name = defaultdict(set) -for bus_stop in bus_stops.values(): - bus_stops_by_name[bus_stop.name].add(bus_stop) -bus_stops_by_name = dict(bus_stops_by_name) + from collections import defaultdict + bus_stops_by_name = defaultdict(set) + for bus_stop in bus_stops.values(): + bus_stops_by_name[bus_stop.name].add(bus_stop) + bus_stops_by_name = dict(bus_stops_by_name) -# ryhmittele bus_stops nimen mukaan -all_clusters = [] -def cluster_bus_stops(): - sorted_bus_stops = sorted(bus_stops.values(), key = lambda bus_stop: bus_stop.name) - for bus_stop in sorted_bus_stops: - if not bus_stop.cluster: - stops_to_cluster = {bus_stop} - # etsi pysäkin samannimiset vastaparit - for pair_candidate in bus_stops_by_name[bus_stop.name]: - distance = pair_candidate.location.etäisyys(bus_stop.location) - if pair_candidate is not bus_stop and distance <= 0.4: - stops_to_cluster.add(pair_candidate) - for stop_to_cluster in stops_to_cluster: - if stop_to_cluster.cluster: - cluster = stop_to_cluster.cluster - break - else: - cluster = BusStopCluster() - all_clusters.append(cluster) - for stop_to_cluster in stops_to_cluster: - if not stop_to_cluster.cluster: - cluster.add_stop(stop_to_cluster) - # Merkitse muistiin pysäkkien vastaparit käyttäen hyväksi tämänhetkistä ryhmittelytietoa - for bus_stop in bus_stops.values(): - if bus_stop.cluster: - bus_stop.pairs = bus_stop.cluster.stops - {bus_stop} - # Ryhmitä ne bus_stops, joilla ei ollut omaa vastaparia, muiden pysäkkien kanssa - for bus_stop in sorted_bus_stops: - if len(bus_stop.cluster.stops) == 1: - possibilities = set() - for cluster in all_clusters: - if cluster is not bus_stop.cluster: - distance = cluster.center.etäisyys(bus_stop.location) - if distance <= 0.4: - possibilities.add((distance, cluster)) - if possibilities: - best = min(possibilities)[1] - all_clusters.remove(bus_stop.cluster) - best.merge(bus_stop.cluster) + # ryhmittele bus_stops nimen mukaan + all_clusters = [] + def cluster_bus_stops(): + sorted_bus_stops = sorted(bus_stops.values(), key = lambda bus_stop: bus_stop.name) + for bus_stop in sorted_bus_stops: + if not bus_stop.cluster: + stops_to_cluster = {bus_stop} + # etsi pysäkin samannimiset vastaparit + for pair_candidate in bus_stops_by_name[bus_stop.name]: + distance = pair_candidate.location.etäisyys(bus_stop.location) + if pair_candidate is not bus_stop and distance <= 0.4: + stops_to_cluster.add(pair_candidate) + for stop_to_cluster in stops_to_cluster: + if stop_to_cluster.cluster: + cluster = stop_to_cluster.cluster + break + else: + cluster = BusStopCluster() + all_clusters.append(cluster) + for stop_to_cluster in stops_to_cluster: + if not stop_to_cluster.cluster: + cluster.add_stop(stop_to_cluster) + # Merkitse muistiin pysäkkien vastaparit käyttäen hyväksi tämänhetkistä ryhmittelytietoa + for bus_stop in bus_stops.values(): + if bus_stop.cluster: + bus_stop.pairs = bus_stop.cluster.stops - {bus_stop} + # Ryhmitä ne bus_stops, joilla ei ollut omaa vastaparia, muiden pysäkkien kanssa + for bus_stop in sorted_bus_stops: + if len(bus_stop.cluster.stops) == 1: + possibilities = set() + for cluster in all_clusters: + if cluster is not bus_stop.cluster: + distance = cluster.center.etäisyys(bus_stop.location) + if distance <= 0.4: + possibilities.add((distance, cluster)) + if possibilities: + best = min(possibilities)[1] + all_clusters.remove(bus_stop.cluster) + best.merge(bus_stop.cluster) -def shared_elements_in_n_sets(sets): - from itertools import combinations - result = set() - for pair in combinations(sets, 2): - result |= pair[0] & pair[1] - return result + def shared_elements_in_n_sets(sets): + from itertools import combinations + result = set() + for pair in combinations(sets, 2): + result |= pair[0] & pair[1] + return result -def name_clusters(): - from collections import defaultdict - from pprint import pprint - clusters_per_name = defaultdict(set) - for cluster in all_clusters: - name_representing_stop = min((len(stop.reference), stop.reference, stop) for stop in cluster.stops)[2] - clusters_per_name[name_representing_stop.name].add(cluster) - for name, clusters in clusters_per_name.items(): - if len(clusters) == 1: - # Ryhmä on ainoa jolla on varaus tälle nimelle. Sen kuin vaan. - next(iter(clusters)).name = name - else: - # Olisiko kaikki klusterit eri alueilla? - common_regions = shared_elements_in_n_sets({stop.region for stop in cluster.stops} for cluster in clusters) - # Esitys: ryhmä -> ne alueet jotka ovat tälle ryhmälle ainutlaatuisia - proposal = { - cluster: {stop.region for stop in cluster.stops} - common_regions - {None} - for cluster in clusters - } - # Jos enintään yksi klusteri tässä esityksessä on kokonaan ilman omaa aluetta, jolla se voisi eritellä, - # niin nimetään klusterit näiden alueiden mukaan. - # Se klusteri jolla ei ole omaa aluetta (jos on) jätetään ilman aluepäätettä. - if sum([1 for unique_areas in proposal.values() if not unique_areas]) <= 1: - for cluster, unique_areas in proposal.items(): - individual_cluster_name = name - if unique_areas: - individual_cluster_name += ' (' + min(unique_areas) + ')' - cluster.name = individual_cluster_name + def name_clusters(): + from collections import defaultdict + from pprint import pprint + clusters_per_name = defaultdict(set) + for cluster in all_clusters: + name_representing_stop = min((len(stop.reference), stop.reference, stop) for stop in cluster.stops)[2] + clusters_per_name[name_representing_stop.name].add(cluster) + for name, clusters in clusters_per_name.items(): + if len(clusters) == 1: + # Ryhmä on ainoa jolla on varaus tälle nimelle. Sen kuin vaan. + next(iter(clusters)).name = name + else: + # Olisiko kaikki klusterit eri alueilla? + common_regions = shared_elements_in_n_sets({stop.region for stop in cluster.stops} for cluster in clusters) + # Esitys: ryhmä -> ne alueet jotka ovat tälle ryhmälle ainutlaatuisia + proposal = { + cluster: {stop.region for stop in cluster.stops} - common_regions - {None} + for cluster in clusters + } + # Jos enintään yksi klusteri tässä esityksessä on kokonaan ilman omaa aluetta, jolla se voisi eritellä, + # niin nimetään klusterit näiden alueiden mukaan. + # Se klusteri jolla ei ole omaa aluetta (jos on) jätetään ilman aluepäätettä. + if sum([1 for unique_areas in proposal.values() if not unique_areas]) <= 1: + for cluster, unique_areas in proposal.items(): + individual_cluster_name = name + if unique_areas: + individual_cluster_name += ' (' + min(unique_areas) + ')' + cluster.name = individual_cluster_name + else: + # Typerä reunatapaus. Indeksoidaan numeroin... + for n, (_, cluster) in enumerate(sorted( + min((stop.reference.lower(), cluster) for stop in cluster.stops) + for cluster in clusters + ), 1): + individual_cluster_name = name + '-' + str(n) + cluster.name = individual_cluster_name + + print('Ryhmitellään pysäkit...') + cluster_bus_stops() + name_clusters() + + for cluster in all_clusters: + if cluster.url_name in clusters_by_name: + print('Warning: Clusters %r and %r share the same URL name: %r' % (cluster.name, clusters_by_name[cluster.url_name].name, cluster.url_name)) else: - # Typerä reunatapaus. Indeksoidaan numeroin... - for n, (_, cluster) in enumerate(sorted( - min((stop.reference.lower(), cluster) for stop in cluster.stops) - for cluster in clusters - ), 1): - individual_cluster_name = name + '-' + str(n) - cluster.name = individual_cluster_name - -print('Ryhmitellään pysäkit...') -cluster_bus_stops() -name_clusters() - -clusters_by_name = {} -for cluster in all_clusters: - if cluster.url_name in clusters_by_name: - print('Warning: Clusters %r and %r share the same URL name: %r' % (cluster.name, clusters_by_name[cluster.url_name].name, cluster.url_name)) - else: - clusters_by_name[cluster.url_name] = cluster + clusters_by_name[cluster.url_name] = cluster -print('Ladataan aikataulut... ', end = '', flush = True, file = stderr) -with open('gtfs/stop_times.txt') as file: - row_count = sum(line.count('\n') for line in file) - progress = 0 - file.seek(0) - for row in read_csv(file): - trip = all_trips[transform_trip_reference(row['trip_id'])] - arrival_time = read_time(row['arrival_time']) - departure_time = read_time(row['departure_time']) - stop = bus_stops[row['stop_id']] - traveled_distance = float(row['shape_dist_traveled']) - trip.schedule.append(BusHalt(arrival_time, departure_time, stop, trip, traveled_distance)) - stop.involved_trips.add(trip) - progress += 1 - if progress % 1000 == 0: - print('\rLadataan aikataulut... %.1f%%' % (progress * 100 / row_count), end = ' ', file = stderr) -print('\rLadataan aikataulut... ladattu', file = stderr) + print('Ladataan aikataulut... ', end = '', flush = True, file = stderr) + with gtfs_zip.open('stop_times.txt') as file: + row_count = sum(line.count(b'\n') for line in file) + with gtfs_zip.open('stop_times.txt') as file: + progress = 0 + for row in read_csv(map(bytes.decode, file)): + trip = all_trips[transform_trip_reference(row['trip_id'])] + arrival_time = read_time(row['arrival_time']) + departure_time = read_time(row['departure_time']) + stop = bus_stops[row['stop_id']] + traveled_distance = float(row['shape_dist_traveled']) * float(profile['metrics']['shape-modifier']) + trip.schedule.append(BusHalt(arrival_time, departure_time, stop, trip, traveled_distance)) + stop.involved_trips.add(trip) + progress += 1 + if progress % 1000 == 0: + print('\rLadataan aikataulut... %.1f%%' % (progress * 100 / row_count), end = ' ', file = stderr) + print('\rLadataan aikataulut... ladattu', file = stderr) + + for trip in all_trips.values(): + from busroute import simplify_name + schedule = trip.concise_schedule() + try: + trip.from_place = simplify_name(schedule[0]) + trip.to_place = simplify_name(schedule[-1]) + except IndexError: + trip.from_place = '' + trip.to_place = '' -for trip in all_trips.values(): - from busroute import simplify_name - schedule = trip.concise_schedule() - try: - trip.from_place = simplify_name(schedule[0]) - trip.to_place = simplify_name(schedule[-1]) - except IndexError: - trip.from_place = '' - trip.to_place = '' + for route in routes.values(): + from collections import Counter + from busroute import simplify_name + tally = Counter() + for trip in route.trips: + schedule = trip.concise_schedule() + places = set(schedule) + do_add = True + assert type(schedule) is list + for candidate in tally: + if places.issubset(set(candidate)): + do_add = False + tally.update({tuple(candidate)}) + if do_add: + tally.update({tuple(schedule)}) + try: + most_common_route = tally.most_common(1)[0][0] + route.description = simplify_name(most_common_route[0]) + ' - ' + simplify_name(most_common_route[-1]) + except: + route.description = '' + route.trips = sorted(route.trips, key = lambda trip: trip.schedule[0].departure_time) -for route in routes.values(): - from collections import Counter - from busroute import simplify_name - tally = Counter() - for trip in route.trips: - schedule = trip.concise_schedule() - places = set(schedule) - do_add = True - assert type(schedule) is list - for candidate in tally: - if places.issubset(set(candidate)): - do_add = False - tally.update({tuple(candidate)}) - if do_add: - tally.update({tuple(schedule)}) - try: - most_common_route = tally.most_common(1)[0][0] - route.description = simplify_name(most_common_route[0]) + ' - ' + simplify_name(most_common_route[-1]) - except: - route.description = '' - route.trips = sorted(route.trips, key = lambda trip: trip.schedule[0].departure_time) + # Fölin datassa on jotain tosi kummaa. Ilmeisesti ajovuoron viimeisen pysähdyksen saapumisaika on ihan täysin + # väärin. Arvaan että se on seuraavan lähdön aika, mutta joka tapauksessa se on väärin. + # Arvataan mikä se todellinen saapumisaika on. Se ei voi mennä kauhean paljon pahemmin vikaan kuin alkuperäinen + # väärin oleva data. + for trip in all_trips.values(): + bus_speed_coefficient = 750 # metriä minuutissa + last_leg_distance = trip.schedule[-1].traveled_distance - trip.schedule[-2].traveled_distance + trip.schedule[-1].arrival_time = trip.schedule[-2].departure_time + timedelta(minutes = last_leg_distance / bus_speed_coefficient) + +if __name__ == '__main__': + from configparser import ConfigParser + profile = ConfigParser() + profile.read('profiles/föli.ini') + load_buses('gtfs.zip', profile)