From 5dd4bd6c25bfb1fb5891d3c95b6deb90a4e31bfa Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 2 Mar 2020 16:53:29 -0800 Subject: [PATCH] write trip matrices require newer pandas add ability to save trips table to pipeline and begin to finish example all time periods fix trip scheduling bug add support for odt skims as well Co-authored-by: Ben Stabler --- activitysim/abm/models/__init__.py | 1 + activitysim/abm/models/trip_matrices.py | 147 ++++++++++ activitysim/abm/models/trip_scheduling.py | 2 +- activitysim/abm/tables/households.py | 18 +- activitysim/abm/test/test_pipeline.py | 21 +- .../example_mtc/configs/settings.yaml | 1 + .../configs/write_trip_matrices.yaml | 274 ++++++++++++++++++ ...p_matrices_annotate_trips_preprocessor.csv | 130 +++++++++ .../example_mtc/configs_mp/settings.yaml | 1 + .../examples/example_mtc/output/.gitignore | 1 + docs/abmexample.rst | 8 + docs/models.rst | 16 + scripts/simulation.py | 1 + setup.py | 2 +- 14 files changed, 616 insertions(+), 7 deletions(-) create mode 100644 activitysim/abm/models/trip_matrices.py create mode 100644 activitysim/examples/example_mtc/configs/write_trip_matrices.yaml create mode 100644 activitysim/examples/example_mtc/configs/write_trip_matrices_annotate_trips_preprocessor.csv diff --git a/activitysim/abm/models/__init__.py b/activitysim/abm/models/__init__.py index c98063af80..a855f1ca74 100644 --- a/activitysim/abm/models/__init__.py +++ b/activitysim/abm/models/__init__.py @@ -27,3 +27,4 @@ from . import trip_purpose from . import trip_purpose_and_destination from . import trip_scheduling +from . import trip_matrices diff --git a/activitysim/abm/models/trip_matrices.py b/activitysim/abm/models/trip_matrices.py new file mode 100644 index 0000000000..baee44aba2 --- /dev/null +++ b/activitysim/abm/models/trip_matrices.py @@ -0,0 +1,147 @@ +# ActivitySim +# See full license in LICENSE.txt. + +import logging + +import openmatrix as omx +import pandas as pd +import numpy as np + +from activitysim.core import config +from activitysim.core import inject +from activitysim.core import pipeline + +from .util import expressions +from .util.expressions import skim_time_period_label + +logger = logging.getLogger(__name__) + + +@inject.step() +def write_trip_matrices(trips, skim_dict, skim_stack): + """ + Write trip matrices step. + + Adds boolean columns to local trips table via annotation expressions, + then aggregates trip counts and writes OD matrices to OMX. Save annotated + trips table to pipeline if desired. + """ + + model_settings = config.read_model_settings('write_trip_matrices.yaml') + trips_df = annotate_trips(trips, skim_dict, skim_stack, model_settings) + + if bool(model_settings.get('SAVE_TRIPS_TABLE')): + pipeline.replace_table('trips', trips_df) + + logger.info('Aggregating trips...') + aggregate_trips = trips_df.groupby(['origin', 'destination'], sort=False).sum() + logger.info('Finished.') + + orig_vals = aggregate_trips.index.get_level_values('origin') + dest_vals = aggregate_trips.index.get_level_values('destination') + + zone_index = pipeline.get_table('land_use').index + assert all(zone in zone_index for zone in orig_vals) + assert all(zone in zone_index for zone in dest_vals) + + _, orig_index = zone_index.reindex(orig_vals) + _, dest_index = zone_index.reindex(dest_vals) + + write_matrices(aggregate_trips, zone_index, orig_index, dest_index, model_settings) + + +def annotate_trips(trips, skim_dict, skim_stack, model_settings): + """ + Add columns to local trips table. The annotator has + access to the origin/destination skims and everything + defined in the model settings CONSTANTS. + + Pipeline tables can also be accessed by listing them under + TABLES in the preprocessor settings. + """ + + trips_df = trips.to_frame() + + trace_label = 'trip_matrices' + + # setup skim keys + assert ('trip_period' not in trips_df) + trips_df['trip_period'] = skim_time_period_label(trips_df.depart) + od_skim_wrapper = skim_dict.wrap('origin', 'destination') + odt_skim_stack_wrapper = skim_stack.wrap(left_key='origin', right_key='destination', + skim_key='trip_period') + skims = { + 'od_skims': od_skim_wrapper, + "odt_skims": odt_skim_stack_wrapper + } + + locals_dict = {} + constants = config.get_model_constants(model_settings) + if constants is not None: + locals_dict.update(constants) + + expressions.annotate_preprocessors( + trips_df, locals_dict, skims, + model_settings, trace_label) + + # Data will be expanded by an expansion weight column from + # the households pipeline table, if specified in the model settings. + hh_weight_col = model_settings.get('HH_EXPANSION_WEIGHT_COL') + + if hh_weight_col and hh_weight_col not in trips_df: + logger.info("adding '%s' from households to trips table" % hh_weight_col) + household_weights = pipeline.get_table('households')[hh_weight_col] + trips_df[hh_weight_col] = trips_df.household_id.map(household_weights) + + return trips_df + + +def write_matrices(aggregate_trips, zone_index, orig_index, dest_index, model_settings): + """ + Write aggregated trips to OMX format. + + The MATRICES setting lists the new OMX files to write. + Each file can contain any number of 'tables', each specified by a + table key ('name') and a trips table column ('data_field') to use + for aggregated counts. + + Any data type may be used for columns added in the annotation phase, + but the table 'data_field's must be summable types: ints, floats, bools. + """ + + matrix_settings = model_settings.get('MATRICES') + + if not matrix_settings: + logger.error('Missing MATRICES setting in write_trip_matrices.yaml') + + for matrix in matrix_settings: + filename = matrix.get('file_name') + filepath = config.output_file_path(filename) + logger.info('opening %s' % filepath) + file = omx.open_file(filepath, 'w') # possibly overwrite existing file + table_settings = matrix.get('tables') + + for table in table_settings: + table_name = table.get('name') + col = table.get('data_field') + + if col not in aggregate_trips: + logger.error('missing %s column in %s DataFrame' % (col, aggregate_trips.name)) + return + + hh_weight_col = model_settings.get('HH_EXPANSION_WEIGHT_COL') + if hh_weight_col: + aggregate_trips[col] = aggregate_trips[col] / aggregate_trips[hh_weight_col] + + data = np.zeros((len(zone_index), len(zone_index))) + data[orig_index, dest_index] = aggregate_trips[col] + logger.info('writing %s' % table_name) + file[table_name] = data # write to file + + # include the index-to-zone map in the file + logger.info('adding %s mapping for %s zones to %s' % + (zone_index.name, zone_index.size, filename)) + file.create_mapping(zone_index.name, zone_index.to_numpy()) + + logger.info('closing %s' % filepath) + file.close() diff --git a/activitysim/abm/models/trip_scheduling.py b/activitysim/abm/models/trip_scheduling.py index 3ca4b890fd..ae63101a23 100644 --- a/activitysim/abm/models/trip_scheduling.py +++ b/activitysim/abm/models/trip_scheduling.py @@ -319,7 +319,7 @@ def schedule_trips_in_leg( trips = trips.sort_index() trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1) is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1) - trips.next_trip_id = trips.next_trip_id.where(is_final, NO_TRIP_ID) + trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID) # iterate over outbound trips in ascending trip_num order, skipping the initial trip # iterate over inbound trips in descending trip_num order, skipping the finial trip diff --git a/activitysim/abm/tables/households.py b/activitysim/abm/tables/households.py index 86c3a21120..6bfc308aa3 100644 --- a/activitysim/abm/tables/households.py +++ b/activitysim/abm/tables/households.py @@ -20,9 +20,11 @@ def households(households_sample_size, override_hh_ids, trace_hh_id): df_full = read_input_table("households") - households_sliced = False + tot_households = df_full.shape[0] + + logger.info("full household list contains %s households" % tot_households) - logger.info("full household list contains %s households" % df_full.shape[0]) + households_sliced = False # only using households listed in override_hh_ids if override_hh_ids is not None: @@ -48,9 +50,9 @@ def households(households_sample_size, override_hh_ids, trace_hh_id): households_sliced = True # if we need a subset of full store - elif households_sample_size > 0 and df_full.shape[0] > households_sample_size: + elif tot_households > households_sample_size > 0: - logger.info("sampling %s of %s households" % (households_sample_size, df_full.shape[0])) + logger.info("sampling %s of %s households" % (households_sample_size, tot_households)) """ Because random seed is set differently for each step, sampling of households using @@ -80,6 +82,14 @@ def households(households_sample_size, override_hh_ids, trace_hh_id): # persons table inject.add_injectable('households_sliced', households_sliced) + if 'sample_rate' not in df.columns: + if households_sample_size == 0: + sample_rate = 1 + else: + sample_rate = round(households_sample_size / tot_households, 3) + + df['sample_rate'] = sample_rate + logger.info("loaded households %s" % (df.shape,)) # FIXME - pathological knowledge of name of chunk_id column used by chunked_choosers_by_chunk_id diff --git a/activitysim/abm/test/test_pipeline.py b/activitysim/abm/test/test_pipeline.py index 1698561b00..9a04c89380 100644 --- a/activitysim/abm/test/test_pipeline.py +++ b/activitysim/abm/test/test_pipeline.py @@ -4,8 +4,12 @@ import logging import pkg_resources +import openmatrix as omx +import numpy as np +import numpy.testing as npt + import pandas as pd -import pandas.util.testing as pdt +import pandas.testing as pdt import pytest import yaml @@ -17,6 +21,7 @@ # set the max households for all tests (this is to limit memory use on travis) HOUSEHOLDS_SAMPLE_SIZE = 100 +HOUSEHOLDS_SAMPLE_RATE = 0.02 # HOUSEHOLDS_SAMPLE_RATE / 5000 households # household with mandatory, non mandatory, atwork_subtours, and joint tours HH_ID = 257341 @@ -52,6 +57,7 @@ def setup_dirs(configs_dir, data_dir=None): tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') + tracing.delete_output_files('omx') def teardown_function(func): @@ -433,6 +439,18 @@ def regress(): # should be at least two tours per trip assert trips_df.shape[0] >= 2*tours_df.shape[0] + # write_trip_matrices + trip_matrices_file = config.output_file_path('trips_md.omx') + assert os.path.exists(trip_matrices_file) + trip_matrices = omx.open_file(trip_matrices_file) + assert trip_matrices.shape() == (25, 25) + + assert 'WALK_MD' in trip_matrices.list_matrices() + walk_trips = np.array(trip_matrices['WALK_MD']) + assert walk_trips.dtype == np.dtype('float64') + + trip_matrices.close() + def test_full_run1(): @@ -517,6 +535,7 @@ def test_full_run5_singleton(): if __name__ == "__main__": + from activitysim import abm # register injectables print("running test_full_run1") test_full_run1() # teardown_function(None) diff --git a/activitysim/examples/example_mtc/configs/settings.yaml b/activitysim/examples/example_mtc/configs/settings.yaml index 9010a163c2..fd688b94b0 100644 --- a/activitysim/examples/example_mtc/configs/settings.yaml +++ b/activitysim/examples/example_mtc/configs/settings.yaml @@ -86,6 +86,7 @@ models: - trip_mode_choice - write_data_dictionary - track_skim_usage + - write_trip_matrices - write_tables # to resume after last successful checkpoint, specify resume_after: _ diff --git a/activitysim/examples/example_mtc/configs/write_trip_matrices.yaml b/activitysim/examples/example_mtc/configs/write_trip_matrices.yaml new file mode 100644 index 0000000000..6059d750e9 --- /dev/null +++ b/activitysim/examples/example_mtc/configs/write_trip_matrices.yaml @@ -0,0 +1,274 @@ +# read trips table post preprocessor and run expressions to code +# additional data fields, with one data fields for each matrix specified below + +preprocessor: + SPEC: write_trip_matrices_annotate_trips_preprocessor + DF: trips + TABLES: + - tours + +# divide trip counts by household expansion factor +HH_EXPANSION_WEIGHT_COL: sample_rate # added when households read in + +# save preprocessed trips table to pipeline if desired +SAVE_TRIPS_TABLE: False + +MATRICES: + - file_name: trips_ea.omx + tables: + - name: DRIVEALONEFREE_EA + data_field: DRIVEALONEFREE_EA + - name: DRIVEALONEPAY_EA + data_field: DRIVEALONEPAY_EA + - name: SHARED2FREE_EA + data_field: SHARED2FREE_EA + - name: SHARED2PAY_EA + data_field: SHARED2PAY_EA + - name: SHARED3FREE_EA + data_field: SHARED3FREE_EA + - name: SHARED3PAY_EA + data_field: SHARED3PAY_EA + - name: WALK_EA + data_field: WALK_EA + - name: BIKE_EA + data_field: BIKE_EA + - name: WALK_LOC_WALK_EA + data_field: WALK_LOC_WALK_EA + - name: WALK_LRF_WALK_EA + data_field: WALK_LRF_WALK_EA + - name: WALK_EXP_WALK_EA + data_field: WALK_EXP_WALK_EA + - name: WALK_HVY_WALK_EA + data_field: WALK_HVY_WALK_EA + - name: WALK_COM_WALK_EA + data_field: WALK_COM_WALK_EA + - name: DRIVE_LOC_WALK_EA + data_field: DRIVE_LOC_WALK_EA + - name: DRIVE_LRF_WALK_EA + data_field: DRIVE_LRF_WALK_EA + - name: DRIVE_EXP_WALK_EA + data_field: DRIVE_EXP_WALK_EA + - name: DRIVE_HVY_WALK_EA + data_field: DRIVE_HVY_WALK_EA + - name: DRIVE_COM_WALK_EA + data_field: DRIVE_COM_WALK_EA + - name: WALK_LOC_DRIVE_EA + data_field: WALK_LOC_DRIVE_EA + - name: WALK_LRF_DRIVE_EA + data_field: WALK_LRF_DRIVE_EA + - name: WALK_EXP_DRIVE_EA + data_field: WALK_EXP_DRIVE_EA + - name: WALK_DRIVE_HVY_EA + data_field: WALK_DRIVE_HVY_EA + - name: WALK_COM_DRIVE_EA + data_field: WALK_COM_DRIVE_EA + - file_name: trips_am.omx + tables: + - name: DRIVEALONEFREE_AM + data_field: DRIVEALONEFREE_AM + - name: DRIVEALONEPAY_AM + data_field: DRIVEALONEPAY_AM + - name: SHARED2FREE_AM + data_field: SHARED2FREE_AM + - name: SHARED2PAY_AM + data_field: SHARED2PAY_AM + - name: SHARED3FREE_AM + data_field: SHARED3FREE_AM + - name: SHARED3PAY_AM + data_field: SHARED3PAY_AM + - name: WALK_AM + data_field: WALK_AM + - name: BIKE_AM + data_field: BIKE_AM + - name: WALK_LOC_WALK_AM + data_field: WALK_LOC_WALK_AM + - name: WALK_LRF_WALK_AM + data_field: WALK_LRF_WALK_AM + - name: WALK_EXP_WALK_AM + data_field: WALK_EXP_WALK_AM + - name: WALK_HVY_WALK_AM + data_field: WALK_HVY_WALK_AM + - name: WALK_COM_WALK_AM + data_field: WALK_COM_WALK_AM + - name: DRIVE_LOC_WALK_AM + data_field: DRIVE_LOC_WALK_AM + - name: DRIVE_LRF_WALK_AM + data_field: DRIVE_LRF_WALK_AM + - name: DRIVE_EXP_WALK_AM + data_field: DRIVE_EXP_WALK_AM + - name: DRIVE_HVY_WALK_AM + data_field: DRIVE_HVY_WALK_AM + - name: DRIVE_COM_WALK_AM + data_field: DRIVE_COM_WALK_AM + - name: WALK_LOC_DRIVE_AM + data_field: WALK_LOC_DRIVE_AM + - name: WALK_LRF_DRIVE_AM + data_field: WALK_LRF_DRIVE_AM + - name: WALK_EXP_DRIVE_AM + data_field: WALK_EXP_DRIVE_AM + - name: WALK_DRIVE_HVY_AM + data_field: WALK_DRIVE_HVY_AM + - name: WALK_COM_DRIVE_AM + data_field: WALK_COM_DRIVE_AM + - file_name: trips_md.omx + tables: + - name: DRIVEALONEFREE_MD + data_field: DRIVEALONEFREE_MD + - name: DRIVEALONEPAY_MD + data_field: DRIVEALONEPAY_MD + - name: SHARED2FREE_MD + data_field: SHARED2FREE_MD + - name: SHARED2PAY_MD + data_field: SHARED2PAY_MD + - name: SHARED3FREE_MD + data_field: SHARED3FREE_MD + - name: SHARED3PAY_MD + data_field: SHARED3PAY_MD + - name: WALK_MD + data_field: WALK_MD + - name: BIKE_MD + data_field: BIKE_MD + - name: WALK_LOC_WALK_MD + data_field: WALK_LOC_WALK_MD + - name: WALK_LRF_WALK_MD + data_field: WALK_LRF_WALK_MD + - name: WALK_EXP_WALK_MD + data_field: WALK_EXP_WALK_MD + - name: WALK_HVY_WALK_MD + data_field: WALK_HVY_WALK_MD + - name: WALK_COM_WALK_MD + data_field: WALK_COM_WALK_MD + - name: DRIVE_LOC_WALK_MD + data_field: DRIVE_LOC_WALK_MD + - name: DRIVE_LRF_WALK_MD + data_field: DRIVE_LRF_WALK_MD + - name: DRIVE_EXP_WALK_MD + data_field: DRIVE_EXP_WALK_MD + - name: DRIVE_HVY_WALK_MD + data_field: DRIVE_HVY_WALK_MD + - name: DRIVE_COM_WALK_MD + data_field: DRIVE_COM_WALK_MD + - name: WALK_LOC_DRIVE_MD + data_field: WALK_LOC_DRIVE_MD + - name: WALK_LRF_DRIVE_MD + data_field: WALK_LRF_DRIVE_MD + - name: WALK_EXP_DRIVE_MD + data_field: WALK_EXP_DRIVE_MD + - name: WALK_DRIVE_HVY_MD + data_field: WALK_DRIVE_HVY_MD + - name: WALK_COM_DRIVE_MD + data_field: WALK_COM_DRIVE_MD + - file_name: trips_pm.omx + tables: + - name: DRIVEALONEFREE_PM + data_field: DRIVEALONEFREE_PM + - name: DRIVEALONEPAY_PM + data_field: DRIVEALONEPAY_PM + - name: SHARED2FREE_PM + data_field: SHARED2FREE_PM + - name: SHARED2PAY_PM + data_field: SHARED2PAY_PM + - name: SHARED3FREE_PM + data_field: SHARED3FREE_PM + - name: SHARED3PAY_PM + data_field: SHARED3PAY_PM + - name: WALK_PM + data_field: WALK_PM + - name: BIKE_PM + data_field: BIKE_PM + - name: WALK_LOC_WALK_PM + data_field: WALK_LOC_WALK_PM + - name: WALK_LRF_WALK_PM + data_field: WALK_LRF_WALK_PM + - name: WALK_EXP_WALK_PM + data_field: WALK_EXP_WALK_PM + - name: WALK_HVY_WALK_PM + data_field: WALK_HVY_WALK_PM + - name: WALK_COM_WALK_PM + data_field: WALK_COM_WALK_PM + - name: DRIVE_LOC_WALK_PM + data_field: DRIVE_LOC_WALK_PM + - name: DRIVE_LRF_WALK_PM + data_field: DRIVE_LRF_WALK_PM + - name: DRIVE_EXP_WALK_PM + data_field: DRIVE_EXP_WALK_PM + - name: DRIVE_HVY_WALK_PM + data_field: DRIVE_HVY_WALK_PM + - name: DRIVE_COM_WALK_PM + data_field: DRIVE_COM_WALK_PM + - name: WALK_LOC_DRIVE_PM + data_field: WALK_LOC_DRIVE_PM + - name: WALK_LRF_DRIVE_PM + data_field: WALK_LRF_DRIVE_PM + - name: WALK_EXP_DRIVE_PM + data_field: WALK_EXP_DRIVE_PM + - name: WALK_DRIVE_HVY_PM + data_field: WALK_DRIVE_HVY_PM + - name: WALK_COM_DRIVE_PM + data_field: WALK_COM_DRIVE_PM + - file_name: trips_ev.omx + tables: + - name: DRIVEALONEFREE_EV + data_field: DRIVEALONEFREE_EV + - name: DRIVEALONEPAY_EV + data_field: DRIVEALONEPAY_EV + - name: SHARED2FREE_EV + data_field: SHARED2FREE_EV + - name: SHARED2PAY_EV + data_field: SHARED2PAY_EV + - name: SHARED3FREE_EV + data_field: SHARED3FREE_EV + - name: SHARED3PAY_EV + data_field: SHARED3PAY_EV + - name: WALK_EV + data_field: WALK_EV + - name: BIKE_EV + data_field: BIKE_EV + - name: WALK_LOC_WALK_EV + data_field: WALK_LOC_WALK_EV + - name: WALK_LRF_WALK_EV + data_field: WALK_LRF_WALK_EV + - name: WALK_EXP_WALK_EV + data_field: WALK_EXP_WALK_EV + - name: WALK_HVY_WALK_EV + data_field: WALK_HVY_WALK_EV + - name: WALK_COM_WALK_EV + data_field: WALK_COM_WALK_EV + - name: DRIVE_LOC_WALK_EV + data_field: DRIVE_LOC_WALK_EV + - name: DRIVE_LRF_WALK_EV + data_field: DRIVE_LRF_WALK_EV + - name: DRIVE_EXP_WALK_EV + data_field: DRIVE_EXP_WALK_EV + - name: DRIVE_HVY_WALK_EV + data_field: DRIVE_HVY_WALK_EV + - name: DRIVE_COM_WALK_EV + data_field: DRIVE_COM_WALK_EV + - name: WALK_LOC_DRIVE_EV + data_field: WALK_LOC_DRIVE_EV + - name: WALK_LRF_DRIVE_EV + data_field: WALK_LRF_DRIVE_EV + - name: WALK_EXP_DRIVE_EV + data_field: WALK_EXP_DRIVE_EV + - name: WALK_DRIVE_HVY_EV + data_field: WALK_DRIVE_HVY_EV + - name: WALK_COM_DRIVE_EV + data_field: WALK_COM_DRIVE_EV + +CONSTANTS: + time_periods: + EA: + first_hour: 3 + last_hour: 5 + AM: + first_hour: 6 + last_hour: 9 + MD: + first_hour: 10 + last_hour: 14 + PM: + first_hour: 15 + last_hour: 18 + EV: + first_hour: 19 + last_hour: 2 diff --git a/activitysim/examples/example_mtc/configs/write_trip_matrices_annotate_trips_preprocessor.csv b/activitysim/examples/example_mtc/configs/write_trip_matrices_annotate_trips_preprocessor.csv new file mode 100644 index 0000000000..8f3e386e60 --- /dev/null +++ b/activitysim/examples/example_mtc/configs/write_trip_matrices_annotate_trips_preprocessor.csv @@ -0,0 +1,130 @@ +Description,Target,Expression +# add additional fields,, +,tour_participants,trips.tour_id.map(tours.number_of_participants) +,distance,od_skims['DIST'] +# code time periods,, +,is_ea,"trips.depart.between(time_periods['EA']['first_hour'], time_periods['EA']['last_hour'])" +,is_am,"trips.depart.between(time_periods['AM']['first_hour'], time_periods['AM']['last_hour'])" +,is_md,"trips.depart.between(time_periods['MD']['first_hour'], time_periods['MD']['last_hour'])" +,is_pm,"trips.depart.between(time_periods['PM']['first_hour'], time_periods['PM']['last_hour'])" +,is_ev,(trips.depart >= time_periods['EV']['first_hour']) | (trips.depart <= time_periods['EV']['last_hour']) +# ea trips,, +,DRIVEALONEFREE_EA,((trips.trip_mode == 'DRIVEALONEFREE') & is_ea) * tour_participants +,DRIVEALONEPAY_EA,((trips.trip_mode == 'DRIVEALONEPAY') & is_ea) * tour_participants +,SHARED2FREE_EA,((trips.trip_mode == 'SHARED2FREE') & is_ea) * tour_participants +,SHARED2PAY_EA,((trips.trip_mode == 'SHARED2PAY') & is_ea) * tour_participants +,SHARED3FREE_EA,((trips.trip_mode == 'SHARED3FREE') & is_ea) * tour_participants +,SHARED3PAY_EA,((trips.trip_mode == 'SHARED3PAY') & is_ea) * tour_participants +,WALK_EA,((trips.trip_mode == 'WALK') & is_ea) * tour_participants +,BIKE_EA,((trips.trip_mode == 'BIKE') & is_ea) * tour_participants +,WALK_LOC_WALK_EA,((trips.trip_mode == 'WALK_LOC') & is_ea) * tour_participants +,WALK_LRF_WALK_EA,((trips.trip_mode == 'WALK_LRF') & is_ea) * tour_participants +,WALK_EXP_WALK_EA,((trips.trip_mode == 'WALK_EXP') & is_ea) * tour_participants +,WALK_HVY_WALK_EA,((trips.trip_mode == 'WALK_HVY') & is_ea) * tour_participants +,WALK_COM_WALK_EA,((trips.trip_mode == 'WALK_COM') & is_ea) * tour_participants +,DRIVE_LOC_WALK_EA,((trips.trip_mode == 'DRIVE_LOC') & is_ea & trips.outbound) * tour_participants +,DRIVE_LRF_WALK_EA,((trips.trip_mode == 'DRIVE_LRF') & is_ea & trips.outbound) * tour_participants +,DRIVE_EXP_WALK_EA,((trips.trip_mode == 'DRIVE_EXP') & is_ea & trips.outbound) * tour_participants +,DRIVE_HVY_WALK_EA,((trips.trip_mode == 'DRIVE_HVY') & is_ea & trips.outbound) * tour_participants +,DRIVE_COM_WALK_EA,((trips.trip_mode == 'DRIVE_COM') & is_ea & trips.outbound) * tour_participants +,WALK_LOC_DRIVE_EA,((trips.trip_mode == 'DRIVE_LOC') & is_ea & ~trips.outbound) * tour_participants +,WALK_LRF_DRIVE_EA,((trips.trip_mode == 'DRIVE_LRF') & is_ea & ~trips.outbound) * tour_participants +,WALK_EXP_DRIVE_EA,((trips.trip_mode == 'DRIVE_EXP') & is_ea & ~trips.outbound) * tour_participants +,WALK_DRIVE_HVY_EA,((trips.trip_mode == 'DRIVE_HVY') & is_ea & ~trips.outbound) * tour_participants +,WALK_COM_DRIVE_EA,((trips.trip_mode == 'DRIVE_COM') & is_ea & ~trips.outbound) * tour_participants +# am trips,, +,DRIVEALONEFREE_AM,((trips.trip_mode == 'DRIVEALONEFREE') & is_am) * tour_participants +,DRIVEALONEPAY_AM,((trips.trip_mode == 'DRIVEALONEPAY') & is_am) * tour_participants +,SHARED2FREE_AM,((trips.trip_mode == 'SHARED2FREE') & is_am) * tour_participants +,SHARED2PAY_AM,((trips.trip_mode == 'SHARED2PAY') & is_am) * tour_participants +,SHARED3FREE_AM,((trips.trip_mode == 'SHARED3FREE') & is_am) * tour_participants +,SHARED3PAY_AM,((trips.trip_mode == 'SHARED3PAY') & is_am) * tour_participants +,WALK_AM,((trips.trip_mode == 'WALK') & is_am) * tour_participants +,BIKE_AM,((trips.trip_mode == 'BIKE') & is_am) * tour_participants +,WALK_LOC_WALK_AM,((trips.trip_mode == 'WALK_LOC') & is_am) * tour_participants +,WALK_LRF_WALK_AM,((trips.trip_mode == 'WALK_LRF') & is_am) * tour_participants +,WALK_EXP_WALK_AM,((trips.trip_mode == 'WALK_EXP') & is_am) * tour_participants +,WALK_HVY_WALK_AM,((trips.trip_mode == 'WALK_HVY') & is_am) * tour_participants +,WALK_COM_WALK_AM,((trips.trip_mode == 'WALK_COM') & is_am) * tour_participants +,DRIVE_LOC_WALK_AM,((trips.trip_mode == 'DRIVE_LOC') & is_am & trips.outbound) * tour_participants +,DRIVE_LRF_WALK_AM,((trips.trip_mode == 'DRIVE_LRF') & is_am & trips.outbound) * tour_participants +,DRIVE_EXP_WALK_AM,((trips.trip_mode == 'DRIVE_EXP') & is_am & trips.outbound) * tour_participants +,DRIVE_HVY_WALK_AM,((trips.trip_mode == 'DRIVE_HVY') & is_am & trips.outbound) * tour_participants +,DRIVE_COM_WALK_AM,((trips.trip_mode == 'DRIVE_COM') & is_am & trips.outbound) * tour_participants +,WALK_LOC_DRIVE_AM,((trips.trip_mode == 'DRIVE_LOC') & is_am & ~trips.outbound) * tour_participants +,WALK_LRF_DRIVE_AM,((trips.trip_mode == 'DRIVE_LRF') & is_am & ~trips.outbound) * tour_participants +,WALK_EXP_DRIVE_AM,((trips.trip_mode == 'DRIVE_EXP') & is_am & ~trips.outbound) * tour_participants +,WALK_DRIVE_HVY_AM,((trips.trip_mode == 'DRIVE_HVY') & is_am & ~trips.outbound) * tour_participants +,WALK_COM_DRIVE_AM,((trips.trip_mode == 'DRIVE_COM') & is_am & ~trips.outbound) * tour_participants +# md trips,, +,DRIVEALONEFREE_MD,((trips.trip_mode == 'DRIVEALONEFREE') & is_md) * tour_participants +,DRIVEALONEPAY_MD,((trips.trip_mode == 'DRIVEALONEPAY') & is_md) * tour_participants +,SHARED2FREE_MD,((trips.trip_mode == 'SHARED2FREE') & is_md) * tour_participants +,SHARED2PAY_MD,((trips.trip_mode == 'SHARED2PAY') & is_md) * tour_participants +,SHARED3FREE_MD,((trips.trip_mode == 'SHARED3FREE') & is_md) * tour_participants +,SHARED3PAY_MD,((trips.trip_mode == 'SHARED3PAY') & is_md) * tour_participants +,WALK_MD,((trips.trip_mode == 'WALK') & is_md) * tour_participants +,BIKE_MD,((trips.trip_mode == 'BIKE') & is_md) * tour_participants +,WALK_LOC_WALK_MD,((trips.trip_mode == 'WALK_LOC') & is_md) * tour_participants +,WALK_LRF_WALK_MD,((trips.trip_mode == 'WALK_LRF') & is_md) * tour_participants +,WALK_EXP_WALK_MD,((trips.trip_mode == 'WALK_EXP') & is_md) * tour_participants +,WALK_HVY_WALK_MD,((trips.trip_mode == 'WALK_HVY') & is_md) * tour_participants +,WALK_COM_WALK_MD,((trips.trip_mode == 'WALK_COM') & is_md) * tour_participants +,DRIVE_LOC_WALK_MD,((trips.trip_mode == 'DRIVE_LOC') & is_md & trips.outbound) * tour_participants +,DRIVE_LRF_WALK_MD,((trips.trip_mode == 'DRIVE_LRF') & is_md & trips.outbound) * tour_participants +,DRIVE_EXP_WALK_MD,((trips.trip_mode == 'DRIVE_EXP') & is_md & trips.outbound) * tour_participants +,DRIVE_HVY_WALK_MD,((trips.trip_mode == 'DRIVE_HVY') & is_md & trips.outbound) * tour_participants +,DRIVE_COM_WALK_MD,((trips.trip_mode == 'DRIVE_COM') & is_md & trips.outbound) * tour_participants +,WALK_LOC_DRIVE_MD,((trips.trip_mode == 'DRIVE_LOC') & is_md & ~trips.outbound) * tour_participants +,WALK_LRF_DRIVE_MD,((trips.trip_mode == 'DRIVE_LRF') & is_md & ~trips.outbound) * tour_participants +,WALK_EXP_DRIVE_MD,((trips.trip_mode == 'DRIVE_EXP') & is_md & ~trips.outbound) * tour_participants +,WALK_DRIVE_HVY_MD,((trips.trip_mode == 'DRIVE_HVY') & is_md & ~trips.outbound) * tour_participants +,WALK_COM_DRIVE_MD,((trips.trip_mode == 'DRIVE_COM') & is_md & ~trips.outbound) * tour_participants +# pm trips,, +,DRIVEALONEFREE_PM,((trips.trip_mode == 'DRIVEALONEFREE') & is_pm) * tour_participants +,DRIVEALONEPAY_PM,((trips.trip_mode == 'DRIVEALONEPAY') & is_pm) * tour_participants +,SHARED2FREE_PM,((trips.trip_mode == 'SHARED2FREE') & is_pm) * tour_participants +,SHARED2PAY_PM,((trips.trip_mode == 'SHARED2PAY') & is_pm) * tour_participants +,SHARED3FREE_PM,((trips.trip_mode == 'SHARED3FREE') & is_pm) * tour_participants +,SHARED3PAY_PM,((trips.trip_mode == 'SHARED3PAY') & is_pm) * tour_participants +,WALK_PM,((trips.trip_mode == 'WALK') & is_pm) * tour_participants +,BIKE_PM,((trips.trip_mode == 'BIKE') & is_pm) * tour_participants +,WALK_LOC_WALK_PM,((trips.trip_mode == 'WALK_LOC') & is_pm) * tour_participants +,WALK_LRF_WALK_PM,((trips.trip_mode == 'WALK_LRF') & is_pm) * tour_participants +,WALK_EXP_WALK_PM,((trips.trip_mode == 'WALK_EXP') & is_pm) * tour_participants +,WALK_HVY_WALK_PM,((trips.trip_mode == 'WALK_HVY') & is_pm) * tour_participants +,WALK_COM_WALK_PM,((trips.trip_mode == 'WALK_COM') & is_pm) * tour_participants +,DRIVE_LOC_WALK_PM,((trips.trip_mode == 'DRIVE_LOC') & is_pm & trips.outbound) * tour_participants +,DRIVE_LRF_WALK_PM,((trips.trip_mode == 'DRIVE_LRF') & is_pm & trips.outbound) * tour_participants +,DRIVE_EXP_WALK_PM,((trips.trip_mode == 'DRIVE_EXP') & is_pm & trips.outbound) * tour_participants +,DRIVE_HVY_WALK_PM,((trips.trip_mode == 'DRIVE_HVY') & is_pm & trips.outbound) * tour_participants +,DRIVE_COM_WALK_PM,((trips.trip_mode == 'DRIVE_COM') & is_pm & trips.outbound) * tour_participants +,WALK_LOC_DRIVE_PM,((trips.trip_mode == 'DRIVE_LOC') & is_pm & ~trips.outbound) * tour_participants +,WALK_LRF_DRIVE_PM,((trips.trip_mode == 'DRIVE_LRF') & is_pm & ~trips.outbound) * tour_participants +,WALK_EXP_DRIVE_PM,((trips.trip_mode == 'DRIVE_EXP') & is_pm & ~trips.outbound) * tour_participants +,WALK_DRIVE_HVY_PM,((trips.trip_mode == 'DRIVE_HVY') & is_pm & ~trips.outbound) * tour_participants +,WALK_COM_DRIVE_PM,((trips.trip_mode == 'DRIVE_COM') & is_pm & ~trips.outbound) * tour_participants +# ev trips,, +,DRIVEALONEFREE_EV,((trips.trip_mode == 'DRIVEALONEFREE') & is_ev) * tour_participants +,DRIVEALONEPAY_EV,((trips.trip_mode == 'DRIVEALONEPAY') & is_ev) * tour_participants +,SHARED2FREE_EV,((trips.trip_mode == 'SHARED2FREE') & is_ev) * tour_participants +,SHARED2PAY_EV,((trips.trip_mode == 'SHARED2PAY') & is_ev) * tour_participants +,SHARED3FREE_EV,((trips.trip_mode == 'SHARED3FREE') & is_ev) * tour_participants +,SHARED3PAY_EV,((trips.trip_mode == 'SHARED3PAY') & is_ev) * tour_participants +,WALK_EV,((trips.trip_mode == 'WALK') & is_ev) * tour_participants +,BIKE_EV,((trips.trip_mode == 'BIKE') & is_ev) * tour_participants +,WALK_LOC_WALK_EV,((trips.trip_mode == 'WALK_LOC') & is_ev) * tour_participants +,WALK_LRF_WALK_EV,((trips.trip_mode == 'WALK_LRF') & is_ev) * tour_participants +,WALK_EXP_WALK_EV,((trips.trip_mode == 'WALK_EXP') & is_ev) * tour_participants +,WALK_HVY_WALK_EV,((trips.trip_mode == 'WALK_HVY') & is_ev) * tour_participants +,WALK_COM_WALK_EV,((trips.trip_mode == 'WALK_COM') & is_ev) * tour_participants +,DRIVE_LOC_WALK_EV,((trips.trip_mode == 'DRIVE_LOC') & is_ev & trips.outbound) * tour_participants +,DRIVE_LRF_WALK_EV,((trips.trip_mode == 'DRIVE_LRF') & is_ev & trips.outbound) * tour_participants +,DRIVE_EXP_WALK_EV,((trips.trip_mode == 'DRIVE_EXP') & is_ev & trips.outbound) * tour_participants +,DRIVE_HVY_WALK_EV,((trips.trip_mode == 'DRIVE_HVY') & is_ev & trips.outbound) * tour_participants +,DRIVE_COM_WALK_EV,((trips.trip_mode == 'DRIVE_COM') & is_ev & trips.outbound) * tour_participants +,WALK_LOC_DRIVE_EV,((trips.trip_mode == 'DRIVE_LOC') & is_ev & ~trips.outbound) * tour_participants +,WALK_LRF_DRIVE_EV,((trips.trip_mode == 'DRIVE_LRF') & is_ev & ~trips.outbound) * tour_participants +,WALK_EXP_DRIVE_EV,((trips.trip_mode == 'DRIVE_EXP') & is_ev & ~trips.outbound) * tour_participants +,WALK_DRIVE_HVY_EV,((trips.trip_mode == 'DRIVE_HVY') & is_ev & ~trips.outbound) * tour_participants +,WALK_COM_DRIVE_EV,((trips.trip_mode == 'DRIVE_COM') & is_ev & ~trips.outbound) * tour_participants diff --git a/activitysim/examples/example_mtc/configs_mp/settings.yaml b/activitysim/examples/example_mtc/configs_mp/settings.yaml index 6439f6924d..1f11116efa 100644 --- a/activitysim/examples/example_mtc/configs_mp/settings.yaml +++ b/activitysim/examples/example_mtc/configs_mp/settings.yaml @@ -87,6 +87,7 @@ models: - trip_mode_choice ### mp_summarize step - write_data_dictionary + - write_trip_matrices - write_tables multiprocess_steps: diff --git a/activitysim/examples/example_mtc/output/.gitignore b/activitysim/examples/example_mtc/output/.gitignore index b987779f45..bf5bf15e3e 100644 --- a/activitysim/examples/example_mtc/output/.gitignore +++ b/activitysim/examples/example_mtc/output/.gitignore @@ -4,3 +4,4 @@ *.h5 *.txt *.yaml +*.omx diff --git a/docs/abmexample.rst b/docs/abmexample.rst index 8b11436171..cd7f84c97e 100644 --- a/docs/abmexample.rst +++ b/docs/abmexample.rst @@ -499,6 +499,9 @@ columns indicates the number of non-mandatory tours by purpose. The current set +------------------------------------------------+--------------------------------------------------------------------+ | :ref:`trip_cbd_parking` | **NOT YET IMPLEMENTED** | +------------------------------------------------+--------------------------------------------------------------------+ +| :ref:`write_trip_matrices` | - write_trip_matrices.yaml | +| | - write_trip_matrices_annotate_trips_preprocessor.csv | ++------------------------------------------------+--------------------------------------------------------------------+ .. index:: chunk_size .. _chunk_size: @@ -567,6 +570,7 @@ The ``models`` setting contains the specification of the data pipeline model ste - trip_mode_choice - write_data_dictionary - track_skim_usage + - write_trip_matrices - write_tables These model steps must be registered orca steps, as noted below. If you provide a ``resume_after`` @@ -747,6 +751,10 @@ The example ``simulation.py`` run model script also writes the final tables to C the :func:`activitysim.core.pipeline.get_table` method via the ``write_tables`` step. This method returns a pandas DataFrame, which is then written to a CSV file by the ``write_tables`` step. +The ``write_trip_matrices`` step processes the trips table to create open matrix (OMX) trip matrices for +assignment. The matrices are configured and coded according to the expressions in the model step +trip annotation file. See :ref:`write_trip_matrices` for more information. + ActivitySim also writes log and trace files to the ``outputs`` folder. The activitysim.log file, which is the overall log file is always produced. If tracing is specified, then trace files are output as well. diff --git a/docs/models.rst b/docs/models.rst index b761b6972a..5e9e3377f5 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -789,6 +789,22 @@ XXXXX function. This function is registered as an orca step in the example Pipe Core Table: ``trips`` | Result Field: ``XXXXX`` | Skims Keys: ``XXXXX`` +.. _write_trip_matrices: + +Write Trip Matrices +------------------- + +Write open matrix (OMX) trip matrices for assignment. Reads the trips table post preprocessor and run expressions +to code additional data fields, with one data fields for each matrix specified. The matrices are scaled by a +household level expansion factor, which is the household sample rate by default, which is calculated when +households are read in at the beginning of a model run. The main interface to write trip +matrices is the :py:func:`~activitysim.abm.models.trip_matrices.write_trip_matrices` function. This function +is registered as an orca step in the example Pipeline. + +Core Table: ``trips`` | Result: ``omx trip matrices`` | Skims Keys: ``origin, destination`` + +.. automodule:: activitysim.abm.models.trip_matrices + :members: .. _utility_steps: diff --git a/scripts/simulation.py b/scripts/simulation.py index 5871b99943..ddc592c244 100644 --- a/scripts/simulation.py +++ b/scripts/simulation.py @@ -29,6 +29,7 @@ def cleanup_output_files(): tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('prof') + tracing.delete_output_files('omx') def run(run_list, injectables=None): diff --git a/setup.py b/setup.py index f61ede7380..329a867f77 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'numpy >= 1.16.1', 'openmatrix >= 0.3.4.1', - 'pandas >= 0.24.1', + 'pandas >= 1.0.1', 'pyyaml >= 5.1', 'tables >= 3.5.1', 'toolz >= 0.8.1',