from otree.api import ( models, widgets, BaseConstants, BaseSubsession, BaseGroup, BasePlayer, Currency as c, currency_range ) import numpy as np from keras.models import * from keras.layers import * from keras.layers.normalization import * from keras.optimizers import * from keras.models import load_model import random import sys sys.path.insert(0,'C:\\Users\\Kerbelev\\oTree\\oTree\\public_goods') import Dqn import tensorflow as tf import xlwt from xlwt import Workbook import datetime #foo.MyClass() #from Dqn import * doc = """ This is a one-period public goods game with 3 players. """ class Constants(BaseConstants): name_in_url = 'public_goods' players_per_group = 3 num_rounds = 26 instructions_template = 'public_goods/Instructions.html' # """Amount allocated to each player""" STORAGE_COST_PER_UNIT = 0.5 BACKORDER_PENALTY_COST_PER_UNIT = 1 # We can play the full game since no actor is programmed to dump stock near end of game ##WEEKS_TO_PLAY = 36 QUEUE_DELAY_WEEKS = 2 INITIAL_STOCK = 12 INITIAL_COST = 0 INITIAL_CURRENT_ORDERS = 0 CUSTOMER_INITIAL_ORDERS = 4 CUSTOMER_SUBSEQUENT_ORDERS = 8 Demand_array=[4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] TARGET_STOCK = 12 MAX_PLACED_ORDER=1000 ##test class Subsession(BaseSubsession): pass##def vars_for_admin_report(self): class Player(BasePlayer): placed_order = models.IntegerField(min=0, max=Constants.MAX_PLACED_ORDER, label="How much will you order?") # if self.id_in_group==1: # placed_order = models.IntegerField(min=0, max = 25, label ="How much will you order?") # elif self.id_in_group==2: # placed_order = models.IntegerField(min=0, max=50, label="How much will you order?") # else: # placed_order = models.IntegerField(min=0, max=100, label="How much will you order?") current_orders = models.IntegerField() received_shipment = models.IntegerField() outgoing_shipment = models.IntegerField() current_stock = models.IntegerField() on_order_inventory = models.IntegerField() currentCost = models.FloatField() backlog=models.IntegerField() incoming_order=models.IntegerField() inventory_level=models.IntegerField() inventory_level_begin=models.IntegerField() inventory_level_final=models.IntegerField() def get_placed_order(self, roundNumber): return self.in_round(roundNumber).placed_order def get_current_orders(self, roundNumber): return self.in_round(roundNumber).current_orders def get_received_shipment(self, roundNumber): return self.in_round(roundNumber).received_shipment def get_outgoing_shipment(self, roundNumber): return self.in_round(roundNumber).outgoing_shipment def get_current_stock(self, roundNumber): return self.in_round(roundNumber).current_stock def get_on_order_inventory(self, roundNumber): return self.in_round(roundNumber).on_order_inventory def get_backlog(self, roundNumber): return self.in_round(roundNumber).backlog def get_incoming_order(self, roundNumber): return self.in_round(roundNumber).incoming_order def get_inventory_level(self, roundNumber): return self.in_round(roundNumber).inventory_level # def calcStateVariables(self): # self.current_orders = 5 # self.received_shipment = 12 # self.current_stock = 3 # self.on_order_inventory = 0 # self.currentCost = 34 def role(self): if self.id_in_group == 1: return 'Retailer' if self.id_in_group == 2: return 'Wholesaler' if self.id_in_group == 4: return 'Distributor' if self.id_in_group == 3: return 'Factory' class LearningAgent(Player): ##Indices for the gameHistory matrix (same as in the learning Code) xinv_level = 0 xreceived_ship = 1 xon_order_inv = 2 xinc_order = 3 xplaced_order = 4 xshipment_out = 5 xbacklog=6 xcurrent_orders=7 xcurrent_stock=8 xcurrentCost=9 inventory_level = 12 current_stock = 12 received_shipment = 4 on_order_inventory = 16 current_orders = 0 placed_order = 4 ##hier output des ANN outgoing_shipment = 4 backlog = 0 incoming_order = 4 currentCost=0 inventory_level_begin = 12 inventory_level_final = 12 id_in_group = 4 def __init__(self): self.stateHistory = np.array([[12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4],[12, 4, 16, 4, 4, 4]]) self.gameHistory=np.zeros((Constants.num_rounds+1, 10)) # json_file=open('model.json', 'r') # loaded_model_json=json_file.read() # json_file.close() # loaded_model=model_from_json(loaded_model_json) # self.model = loaded_model # self.model=Sequential() # self.model.add(Dense(30, input_dim=30,activation='relu')) # self.model.add(Dense(70,activation='relu')) # self.model.add(Dense(12)) #self.model=load_model("C:/Users/Kerbelev/learningGame/NN30_70.h5") #self.model.load_weights("C:/Users/Kerbelev/learningGame/dqn_beergame.h5") #self.model=Dqn.DQN((30,),12,1e-3) #self.model.load_model("C:/Users/Kerbelev/learningGame/dqn_beergame.h5") self.model=Dqn.DQN((30,),12,1e-3) self.model.load_model("C:/Users/Kerbelev/learningGame/dqn_beergame_ownCostReward04060106.h5") global graph graph = tf.get_default_graph() #print(self.gameHistory) #self.id_in_group=4 #print(self.stateHistory, "Konstruktor") return def updateStateHistory(self, weekNum): for i in range(5): # If we want to look further in the history, adjust i to a larger number if weekNum - i < 0: return for j in range(6): self.stateHistory[i][j] = self.gameHistory[weekNum - i][j] return def get_action(self, state): with graph.as_default(): prediction = self.model.predict(state) argmax = np.argmax(prediction) return argmax def calculate_amount_to_order(self): state=self.stateHistory state = np.reshape(state, (1, state.shape[0] * state.shape[1])) print("this is the tested state: ", state) print("the corresponding shape: ", state.shape) action=self.get_action(state) # switcher = { # 0: np.round(self.incoming_order*0,0), # 1: np.round(self.incoming_order*0.25,0), # 2: np.round(self.incoming_order*0.5,0), # 3: np.round(self.incoming_order*0.75,0), # 4: np.round(self.incoming_order,0), # 5: np.round(self.incoming_order*1.25,0), # 6: np.round(self.incoming_order*1.5,0), # 7: np.round(self.incoming_order*1.75,0), # 8: np.round(self.incoming_order*2,0) , # 9: np.round(self.incoming_order*2.5,0), # 10:np.round(self.incoming_order*3,0), # 11:np.round(self.incoming_order*4,0) # } switcher = { 0: 0, 1: 1, 2: 3, 3: 6, 4: 10, 5: 15, 6: 21, 7: 28, 8: 36, 9: 45, 10:55, 11:66, 12:78 } amountToOrder=switcher.get(action) return amountToOrder def updateGameHistory(self,roundNumber): print("inv level: ", self.current_stock) self.gameHistory[roundNumber][self.xinv_level]=self.inventory_level self.gameHistory[roundNumber][self.xreceived_ship] = self.received_shipment self.gameHistory[roundNumber][self.xon_order_inv] = self.on_order_inventory self.gameHistory[roundNumber][self.xinc_order] = self.incoming_order self.gameHistory[roundNumber][self.xplaced_order] = self.placed_order self.gameHistory[roundNumber][self.xshipment_out] = self.outgoing_shipment self.gameHistory[roundNumber][self.xbacklog] = self.backlog self.gameHistory[roundNumber][self.xcurrent_orders] = self.current_orders self.gameHistory[roundNumber][self.xcurrent_stock] = self.current_stock self.gameHistory[roundNumber][self.xcurrentCost]=self.currentCost def get_placed_order(self, roundNumber): ##added -1 in each round_number from working status return self.gameHistory[roundNumber-1][self.xplaced_order] def get_current_orders(self, roundNumber): return self.gameHistory[roundNumber-1][self.xinc_order] def get_received_shipment(self, roundNumber): return self.gameHistory[roundNumber-1][self.xreceived_ship] def get_outgoing_shipment(self, roundNumber): return self.gameHistory[roundNumber-1][self.xshipment_out] def get_current_stock(self, roundNumber): return self.gameHistory[roundNumber-1][self.xcurrent_stock]## changed index from xinv_level to xcurrent_stock def get_on_order_inventory(self, roundNumber): return self.gameHistory[roundNumber-1][self.xon_order_inv] def get_backlog(self, roundNumber): return self.gameHistory[roundNumber-1][self.xbacklog] def get_current_orders(self, roundNumber): return self.gameHistory[roundNumber-1][self.xcurrent_orders] def get_current_cost(self, roundNumber): return self.gameHistory[roundNumber-1][self.xcurrentCost] class Group(BaseGroup): #SupplyChainActors = {"armin": 1, "ben": 2, "jan": 1} learningAgent=LearningAgent() def get_player(self,player_id): if player_id ==3: return self.learningAgent elif player_id<=2: return self.get_player_by_id(player_id) else: return self.get_player_by_id(3) # def calcPlayersStateVariables (self,p): # if p.id_in_group<=2: # player_id=p.id_in_group # elif p.id_in_group==3: # player_id=4 # else: # player_id=3 # # if player_id == 1: # p.current_orders = Constants.Demand[self.round_number - 3] + p.in_round(self.round_number - 1).backlog # else: # p.current_orders = self.get_player_by_id(player_id-1).in_round(self.round_number - 3).placed_order + p.in_round(self.round_number - 1).backlog # if player_id <4: # p.received_shipment = self.get_player_by_id(player_id+1).in_round(self.round_number - 3).outgoing_shipment # else: # p.received_shipment = self.get_player_by_id(player_id).in_round( # self.round_number - 2).placed_order # current_stock_begin = p.in_round(self.round_number - 1).current_stock + p.received_shipment # p.outgoing_shipment = min(p.current_orders, current_stock_begin) # p.backlog = p.current_orders - p.outgoing_shipment # p.current_stock = current_stock_begin - p.outgoing_shipment # if player_id<4: # p.on_order_inventory = self.get_player_by_id(player_id).in_round( # self.round_number - 1).on_order_inventory + self.get_player_by_id(player_id).in_round( # self.round_number - 1).placed_order - p.received_shipment # else: # p.on_order_inventory = self.get_player_by_id(player_id).in_round( # self.round_number - 2).placed_order # # p.currentCost=p.in_round(self.round_number-1).currentCost + p.current_stock+ 2*p.backlog # # ##to show the real inventory to players on the screen # p.inventory_level=p.current_stock-p.backlog # #print("player id: ", p.id_in_group) def calcPlayersStateVariables(self, p): if p.id_in_group <= 2: player_id = p.id_in_group elif p.id_in_group == 3: player_id = 4 else: player_id = 3 if player_id < 4: ##Vorschlag: if self.round_number<=2: ## p.received_shipment=4 ##else: p.received_shipment = self.get_player(player_id + 1).get_outgoing_shipment(self.round_number-2) else: p.received_shipment = self.get_player(player_id).get_placed_order(self.round_number-2) current_stock_begin = p.get_current_stock(self.round_number - 1) + p.received_shipment if player_id == 1: # if self.round_number<=4: # Demand=4 # else: # Demand = int(round(np.random.uniform(8,10),0)) p.current_orders = Constants.Demand_array[self.round_number-1] + p.in_round(self.round_number - 1).backlog p.incoming_order= Constants.Demand_array[self.round_number-1] else: p.current_orders = self.get_player(player_id - 1).get_placed_order(self.round_number - 2) + p.get_backlog(self.round_number - 1) p.incoming_order=self.get_player(player_id - 1).get_placed_order(self.round_number - 2) p.outgoing_shipment = min(p.current_orders, current_stock_begin) p.backlog = p.current_orders - p.outgoing_shipment p.current_stock = current_stock_begin - p.outgoing_shipment if player_id < 4: p.on_order_inventory = self.get_player(player_id).get_on_order_inventory(self.round_number - 1) + self.get_player(player_id).get_placed_order(self.round_number - 1) - p.received_shipment if player_id==3: print("on_order in t-1 of distributor is: ", self.get_player(player_id).get_on_order_inventory(self.round_number - 1)) print("placed order last round by Distr. is: ",self.get_player(player_id).get_placed_order(self.round_number - 1)) print("Received shipment of distr. is: ", p.received_shipment) else: p.on_order_inventory = self.get_player(player_id).get_placed_order(self.round_number - 1) if player_id!=3: p.currentCost = p.in_round(self.round_number - 1).currentCost + p.current_stock*0.5 + 1 * p.backlog else: p.currentCost = p.get_current_cost(self.round_number-1)+ p.current_stock*0.5 + 1 * p.backlog ##to show the real inventory to players on the screen p.inventory_level = p.current_stock - p.backlog # print("player id: ", p.id_in_group) if player_id!=3: p.inventory_level_begin=self.get_player(player_id).get_inventory_level(self.round_number-1) p.inventory_level_final=p.inventory_level_begin+p.received_shipment-p.incoming_order if player_id==3: self.updateGameHistory() self.learningAgent.updateStateHistory(self.round_number-1) self.learningAgent.placed_order=self.learningAgent.calculate_amount_to_order() print("Learning Agent placed order: ", self.learningAgent.placed_order) self.updatePlacedOrderInGameHistory(self.learningAgent.placed_order,self.round_number-1) def updatePlacedOrderInGameHistory(self, plcdOrder,roundNumber): self.learningAgent.gameHistory[roundNumber][self.learningAgent.xplaced_order] = plcdOrder def calcStateVariables(self,p): # print(self.learningAgent.stateHistory) # global model # model = Dqn.DQN((30,), 12, 1e-3) # model.load_model("C:/Users/Kerbelev/learningGame/dqn_beergame.h5") # #model._make_predict_function() # global graph # graph = tf.get_default_graph() # matrix = [[12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4], [12, 4, 16, 4, 4, 4], # [12, 4, 16, 4, 4, 4]] # numpy_matrix = np.array(matrix) # print(numpy_matrix) # print(numpy_matrix.shape) # new_matrix = np.reshape(numpy_matrix, (1, 30)) # print(new_matrix) # print(new_matrix.shape) # with graph.as_default(): # x = model.predict(new_matrix) # print("result is: ", x) # z = 9 if self.round_number <= 2: self.initializeRound(p) if p.id_in_group==2: self.initializeRound(self.learningAgent) self.updateGameHistory() else: self.calcPlayersStateVariables(p) ## if p.id_in_group==2: #print("before:") #print(self.learningAgent.gameHistory) #self.updateGameHistory() #print("After:") #print(self.learningAgent.gameHistory) self.calcPlayersStateVariables(self.learningAgent) #print("After:") #print(self.learningAgent.gameHistory) if self.round_number==Constants.num_rounds: self.saveGameHistoryToExcel() print(self.learningAgent.gameHistory) def initializeRound(self,p): p.incoming_order = 4 p.current_orders = 4 p.current_stock = 12 p.outgoing_shipment = 4 p.received_shipment = 4 if p.id_in_group != 3: p.on_order_inventory = 16 else: p.on_order_inventory = 4 p.backlog = 0 p.inventory_level = p.current_stock - p.backlog if self.round_number == 1: p.currentCost = 0 elif self.round_number == 2: p.currentCost = 6 p.inventory_level_begin=12 p.inventory_level_final=12 def updateGameHistory(self): print(" round number ",self.round_number) self.learningAgent.updateGameHistory(self.round_number-1) def saveGameHistoryToExcel(self): # Workbook is created wb = Workbook() # add_sheet is used to create sheet. sheet1 = wb.add_sheet('Sheet 1') sheet1.write(0, 1, 'Inventory level') sheet1.write(0, 2, 'Received Shipment') sheet1.write(0, 3, 'On_order_inventory') sheet1.write(0, 4, 'Incoming Order') sheet1.write(0, 5, 'Placed Order') sheet1.write(0, 6, 'Outgoing Shipment') sheet1.write(0, 7, 'Cost') for i in range(1,Constants.num_rounds+1): for j in range (1,8): if j<=6: sheet1.write(i,j,self.learningAgent.gameHistory[i-1][j-1]) elif j==7: sheet1.write(i, j, self.learningAgent.gameHistory[i - 1][j + 2]) now = datetime.datetime.now() now_str = now.strftime("%m%d%H%M") wb.save(now_str+'gameHistoryDistributor.xls')