train_dqn.py

import pandas as pd
import _pickle as pickle
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 
from dqn_v3 import Agent

import sys
sys.path.append('/home/wenqi/HybridDelivery')
sys.path.append('/home/wenqi/Ashutosh')

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--experiment")
args = parser.parse_args()
from utils.utils import Params
params = Params(args.experiment)
if params.scenario == ('DeepPool' or 'GD'):
    from simulator_DeepPool import FleetSimulator
    from debug_experiment import run, load_trip_chunks, describe
    print("           Imported DeepPool Simulator           ")
    
elif params.scenario == 'Hybrid':
    print("           Imported FlexPool Simulator           ")
    from hybrid_experiment import run, load_trip_chunks, describe
    from simulator_Hybrid import FleetSimulator
else:
    print("           Imported MultiHop Simulator           ")
    from simulator_MHRS import FleetSimulator
    from hybrid_experiment import run, load_trip_chunks, describe

GRAPH_PATH = params.GRAPH_PATH
TRIP_PATH = params.TRIP_PATH
ETA_MODEL_PATH = params.ETA_MODEL_PATH
GEOHASH_TABLE_PATH = params.GEOHASH_TABLE_PATH
SCORE_PATH = params.SCORE_PATH
INITIAL_MEMORY_PATH = '/home/wenqi/Ashutosh/' + params.INITIAL_MEMORY_PATH

INITIAL_MEMORY = bool(params.INITIAL_MEMORY)
LOAD_NETWORK = bool(params.LOAD_NETWORK)

NUM_TRIPS = params.NUM_TRIPS
DURATION = params.DURATION
NUM_FLEETS = params.NUM_FLEETS


NO_OP_STEPS = params.NO_OP_STEPS  # Number of "do nothing" actions to be performed by the agent at the start of an episode
CYCLE = params.CYCLE
ACTION_UPDATE_CYCLE = params.ACTION_UPDATE_CYCLE
DEMAND_FORECAST_INTERVAL = params.DEMAND_FORECAST_INTERVAL
AVERAGE_CYCLE = params.AVERAGE_CYCLE
NUM_EPISODES = params.NUM_EPISODES

def main():
    print("Loading models...")
    with open(GRAPH_PATH, 'rb') as f:
        G = pickle.load(f)
    with open(ETA_MODEL_PATH, 'rb') as f:
        eta_model = pickle.load(f)
    num_fleets = NUM_FLEETS

    geohash_table = pd.read_csv(GEOHASH_TABLE_PATH, index_col='geohash')

    env = FleetSimulator(params, G, eta_model, CYCLE, ACTION_UPDATE_CYCLE)
    #assert params.scenario == env.scenario, "\nEnsure that the right simulator is loaded for this experiment scenario!\n"

    agent = Agent(params, geohash_table, CYCLE, ACTION_UPDATE_CYCLE, DEMAND_FORECAST_INTERVAL,
                  training=True, load_network=LOAD_NETWORK)
    if INITIAL_MEMORY:
        with open(INITIAL_MEMORY_PATH, 'rb') as f:
            ex_memory = pickle.load(f)
        agent.init_train(100, ex_memory,summary_duration=100)
    
    

    trip_chunks = load_trip_chunks(params.scenario, TRIP_PATH, NUM_TRIPS, DURATION)[:NUM_EPISODES]
    ####### EDITED HERE #######
    print("!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!")
    print("Running scenario: ",params.scenario)
    print("Experiment ID: : ",params.id)
    print("!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!~!")
    #pdb.set_trace()
    ####### EDITED HERE #######
    print("CHUNKS: ", len(trip_chunks))
    experiment_id = params.scenario+params.id
    spath = SCORE_PATH+"/"+experiment_id
    if not os.path.exists(spath):
        os.makedirs(spath)

    summary_log_path = spath + "/"+params.scenario+"-summary" + '.txt' 
    f=open(summary_log_path, "w+")
    f.close()

    try:
        ds = params.DOWN_SAMPLE
        print("DOWN SAMPLING REQUESTS by a factor of ",ds)
    except:
        ds = 1 

        
    for episode, (trips, date, dayofweek, minofday) in enumerate(trip_chunks):
        # num_fleets = int(np.sqrt(len(trips)/120000.0) * NUM_FLEETS)
        env.reset(num_fleets, trips, dayofweek, minofday)
        if params.scenario=='Hybrid':
            _, requests, _, _, _,_,_,_,_= env.step()
        else:
            _, requests, _, _, _,_,_,_= env.step()
        agent.reset(requests, env.dayofweek, env.minofday)
        num_steps = int(DURATION / CYCLE - NO_OP_STEPS)

        print("#############################################################################")
        print("EPISODE: {:d} / DATE: {:d} / DAYOFWEEK: {:d} / MINUTES: {:.1f} / VEHICLES: {:d}".format(
            episode, date, env.dayofweek, env.minofday, num_fleets
        ))
        
        score, _ = run(experiment_id, env, agent, num_steps, average_cycle=AVERAGE_CYCLE, cheat=True, down_sample=ds)
        env.REJECT_DISTANCE = params.REJECT_DISTANCE
        env.out_of_range = 0

        score_path = spath + "/episode_"+ str(episode) + '.csv'

        
        describe(score, episode, summary_log_path)
        score.to_csv(score_path)

        if episode >= 0 and episode % 2 == 0:
            #print("Saving Experience Memory: {:d}").format(episode)
            #######EDITING BELOW
            with open('/home/wenqi/Ashutosh/' + 'ex_memory_v7' + str(episode) + '.pkl', 'wb') as f:
                pickle.dump(agent.replay_memory, f)


if __name__ == '__main__':
    main()