From 07fabe30e20cd4b7fd3402d1f5d0272925d28095 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Fri, 5 Jan 2024 14:24:17 -0800 Subject: [PATCH 01/26] added context manager and updated to use cim-graph 1.0 --- .../field_interface/agents/agents.py | 47 +-- .../field_interface/context_manager.py | 280 ++++++++++++++++++ gridappsd-field-bus-lib/pyproject.toml | 6 +- gridappsd-python-lib/pyproject.toml | 2 +- 4 files changed, 313 insertions(+), 22 deletions(-) create mode 100644 gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py index d5041b0..c6ab326 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py @@ -6,9 +6,10 @@ from datetime import datetime from typing import Dict -from cimgraph.loaders import ConnectionParameters, gridappsd -from cimgraph.loaders.gridappsd import GridappsdConnection -from cimgraph.models import DistributedModel, SecondaryArea, SwitchArea +from cimgraph.databases import ConnectionParameters +from cimgraph.databases.gridappsd import GridappsdConnection +from cimgraph.models import FeederModel +from cimgraph.models.distributed_area import DistributedArea from gridappsd import DifferenceBuilder import gridappsd.topics as t @@ -17,16 +18,20 @@ from gridappsd.field_interface.interfaces import (FieldMessageBus, MessageBusDefinition) +CIM_PROFILE = None +IEC61970_301 = None cim = None -sparql = None _log = logging.getLogger(__name__) -def set_cim_profile(cim_profile): +def set_cim_profile(cim_profile:str, iec61970_301:int): + global CIM_PROFILE + global IEC61970_301 global cim + CIM_PROFILE = cim_profile + IEC61970_301 = iec61970_301 cim = importlib.import_module('cimgraph.data_profile.' + cim_profile) - gridappsd.set_cim_profile(cim_profile) @dataclass @@ -58,8 +63,10 @@ def __init__(self, self.context = None #TODO: Change params and connection to local connection - self.params = ConnectionParameters() + self.params = ConnectionParameters(cim_profile=CIM_PROFILE, iec61970_301=IEC61970_301) + self.connection = GridappsdConnection(self.params) + self.connection.cim_profile = cim_profile self.app_id = agent_config['app_id'] self.description = agent_config['description'] @@ -263,10 +270,11 @@ def __init__(self, self._connect() if self.agent_area_dict is not None: - feeder = cim.Feeder(mRID=self.downstream_message_bus_def.id) - self.feeder_area = DistributedModel(connection=self.connection, - feeder=feeder, - topology=self.agent_area_dict) + feeder = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + self.feeder_area = DistributedArea(connection=self.connection, + container=feeder, + distributed=True) + self.feeder_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) class SwitchAreaAgent(DistributedAgent): @@ -285,9 +293,11 @@ def __init__(self, self._connect() if self.agent_area_dict is not None: - self.switch_area = SwitchArea(self.downstream_message_bus_def.id, - self.connection) - self.switch_area.initialize_switch_area(self.agent_area_dict) + container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + self.switch_area = DistributedArea(container=container, + connection=self.connection, + distributed=True) + self.switch_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) class SecondaryAreaAgent(DistributedAgent): @@ -308,10 +318,11 @@ def __init__(self, if self.agent_area_dict is not None: if len(self.agent_area_dict['addressable_equipment']) == 0: _log.warn(f"No addressable equipment in the secondary area with down stream message bus id: {self.downstream_message_bus.id}.") - - self.secondary_area = SecondaryArea(self.downstream_message_bus_def.id, - self.connection) - self.secondary_area.initialize_secondary_area(self.agent_area_dict) + container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + self.secondary_area = DistributedArea(container=container, + connection=self.connection, + distributed=True) + self.secondary_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py new file mode 100644 index 0000000..8481bcb --- /dev/null +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py @@ -0,0 +1,280 @@ +import argparse +import logging +import os +import time +from typing import Dict + +import gridappsd.field_interface.agents.agents as agents_mod +import gridappsd.topics as t +from cimgraph.data_profile import CIM_PROFILE +from gridappsd import GridAPPSD +from gridappsd.field_interface.agents import (FeederAgent, SecondaryAreaAgent, + SwitchAreaAgent) +from gridappsd.field_interface.interfaces import MessageBusDefinition + + +cim_profile = CIM_PROFILE.RC4_2021.value + +agents_mod.set_cim_profile(cim_profile=cim_profile,iec61970_301=7) + +cim = agents_mod.cim + +logging.basicConfig(level=logging.DEBUG) +logging.getLogger('goss').setLevel(logging.ERROR) +logging.getLogger('stomp.py').setLevel(logging.ERROR) + +_log = logging.getLogger(__name__) + +#FieldBusManager's request topics. To be used only by context manager user role only. +REQUEST_FIELD = ".".join((t.PROCESS_PREFIX, "request.field")) + + +class FeederAreaContextManager(FeederAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + feeder_dict: Dict = None, + simulation_id: str = None): + + self.ot_connection = GridAPPSD() + if feeder_dict is None: + request = {'request_type':'get_context', + 'modelId': downstream_message_bus_def.id} + feeder_dict = None + while feeder_dict is None: + self.ot_connection.get_logger().info(f"Requesting topology for {self.__class__}") + response = self.ot_connection.get_response(REQUEST_FIELD, request, timeout=10) + if 'data' in response: + feeder_dict = response['data'] + self.ot_connection.send_status("******RCVD FEEDER *************", '/topic/goss.gridappsd.platform.log', 'DEBUG') + else: + time.sleep(5) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, feeder_dict, simulation_id) + + #Override agent_id to a static value + self.agent_id = downstream_message_bus_def.id + '.context_manager' + + self.context = None + + self.registered_agents = {} + self.registered_agents[self.agent_id] = self.get_registration_details() + + self.neighbouring_agents = {} + self.upstream_agents = {} + self.downstream_agents = {} + + def on_request(self, message_bus, headers: Dict, message): + + _log.debug(f"Received request: {message}") + + if message['request_type'] == 'get_context': + reply_to = headers['reply-to'] + if self.context is None: + self.context = self.ot_connection.get_response(REQUEST_FIELD, message) + message_bus.send(reply_to,self.context) + + elif message['request_type'] == 'register_agent': + self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) + self.registered_agents[message['agent'] + ['agent_id']] = message['agent'] + + elif message['request_type'] == 'get_agents': + reply_to = headers['reply-to'] + message_bus.send(reply_to, self.registered_agents) + + +class SwitchAreaContextManager(SwitchAreaAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + switch_area_dict: Dict = None, + simulation_id: str = None): + + self.ot_connection = GridAPPSD() + if switch_area_dict is None: + request = {'request_type':'get_context', + 'areaId': downstream_message_bus_def.id} + switch_area_dict = self.ot_connection.get_response( + REQUEST_FIELD, request, timeout=10)['data'] + + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, switch_area_dict, simulation_id) + + #Override agent_id to a static value + self.agent_id = downstream_message_bus_def.id + '.context_manager' + + self.context = None + + self.registered_agents = {} + self.registered_agents[self.agent_id] = self.get_registration_details() + + def on_request(self, message_bus, headers: Dict, message): + + _log.debug(f"Received request: {message}") + + if message['request_type'] == 'get_context': + #TODO: check for initialization + reply_to = headers['reply-to'] + if self.context is None: + self.context = self.ot_connection.get_response(REQUEST_FIELD, message) + message_bus.send(reply_to,self.context) + + elif message['request_type'] == 'register_agent': + #TODO: check for initialization + self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) + self.registered_agents[message['agent'] + ['agent_id']] = message['agent'] + + elif message['request_type'] == 'get_agents': + #TODO: check for initialization + reply_to = headers['reply-to'] + message_bus.send(reply_to, self.registered_agents) + + elif message['request_type'] == 'is_initialized': + reply_to = headers['reply-to'] + message = {'is_initialized':self.is_initialized} + message_bus.send(reply_to, message) + + +class SecondaryAreaContextManager(SecondaryAreaAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + secondary_area_dict: Dict = None, + simulation_id: str = None): + + self.ot_connection = GridAPPSD() + if secondary_area_dict is None: + request = {'request_type':'get_context', + 'areaId': downstream_message_bus_def.id} + secondary_area_dict = self.ot_connection.get_response( + REQUEST_FIELD, request, timeout=10)['data'] + + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, secondary_area_dict, simulation_id) + + #Override agent_id to a static value + self.agent_id = downstream_message_bus_def.id + '.context_manager' + + self.context = None + + self.registered_agents = {} + self.registered_agents[self.agent_id] = self.get_registration_details() + + def on_request(self, message_bus, headers: Dict, message): + + _log.debug(f"Received request: {message}") + _log.debug(f"Received request: {headers}") + + if message['request_type'] == 'get_context': + reply_to = headers['reply-to'] + if self.context is None: + self.context = self.ot_connection.get_response(REQUEST_FIELD, message) + message_bus.send(reply_to,self.context) + + + elif message['request_type'] == 'register_agent': + self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) + self.registered_agents[message['agent'] + ['agent_id']] = message['agent'] + + elif message['request_type'] == 'get_agents': + reply_to = headers['reply-to'] + message_bus.send(reply_to, self.registered_agents) + +def get_MessageBusDefinition(area_id: str) -> MessageBusDefinition: + + connection_args = {"GRIDAPPSD_ADDRESS": os.environ.get('GRIDAPPSD_ADDRESS',"tcp://gridappsd:61613"), + "GRIDAPPSD_USER": os.environ.get('GRIDAPPSD_USER'), + "GRIDAPPSD_PASSWORD": os.environ.get('GRIDAPPSD_PASSWORD'), + "GRIDAPPSD_APPLICATION_ID": os.environ.get('GRIDAPPSD_APPLICATION_ID')} + + bus = MessageBusDefinition(id=area_id, + is_ot_bus= True, + connection_type = "GRIDAPPSD_TYPE_GRIDAPPSD", + conneciton_args = connection_args) + + return bus + +def _main(): + + time.sleep(10) + parser = argparse.ArgumentParser() + parser.add_argument( + "--simulation_id", + help= + "Simulation id to use for communicating with simulated devices on the message bus. \ + If simulation_id is not provided then Context Manager assumes to run on deployed field with real devices.", + required=False) + opts = parser.parse_args() + simulation_id = opts.simulation_id + + agent_config = { + "app_id": + "context_manager", + "description": + "This agent provides topological context information like neighboring agents and devices to other distributed agents" + } + + + gapps = GridAPPSD() + response = gapps.get_response(t.PLATFORM_STATUS, {"isField":True}) + field_model_mrid = response['fieldModelMrid'] + + is_field_initialized = False + + while not is_field_initialized: + response = gapps.get_response(REQUEST_FIELD, {"request_type":"is_initilized"}) + print(response) + is_field_initialized = response['data']['initialized'] + time.sleep(1) + + system_message_bus_def = get_MessageBusDefinition(field_model_mrid) + feeder_message_bus_def = get_MessageBusDefinition(field_model_mrid) + + #TODO: create access control for agents for different layers + feeder_agent = FeederAreaContextManager(system_message_bus_def, + feeder_message_bus_def, + agent_config, + simulation_id=simulation_id) + + + + for switch_area in feeder_agent.agent_area_dict['switch_areas']: + switch_area_message_bus_def = get_MessageBusDefinition(str(switch_area['message_bus_id'])) + print("Creating switch area agent " + + str(switch_area['message_bus_id'])) + switch_area_agent = SwitchAreaContextManager( + feeder_message_bus_def, + switch_area_message_bus_def, + agent_config, + simulation_id=simulation_id) + + # create secondary area distributed agents + for secondary_area in switch_area['secondary_areas']: + secondary_area_message_bus_def = get_MessageBusDefinition(str(secondary_area['message_bus_id'])) + print("Creating secondary area agent " + + str(secondary_area['message_bus_id'])) + secondary_area_agent = SecondaryAreaContextManager( + switch_area_message_bus_def, + secondary_area_message_bus_def, + agent_config, + simulation_id=simulation_id) + + while True: + try: + time.sleep(0.1) + except KeyboardInterrupt: + print("Exiting sample") + break + + +if __name__ == "__main__": + _main() diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 6da09ad..9207b26 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-field-bus" -version = "2023.5.2a0" +version = "2024.1.1a0" description = "GridAPPS-D Field Bus Implementation" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -23,9 +23,9 @@ packages = [ [tool.poetry.dependencies] -python = ">=3.7.9,<4.0" +python = ">=3.8.1,<4.0" gridappsd-python = {path="../gridappsd-python-lib", develop=true} -cim-graph = "^2023.5.1a3" +cim-graph = "^0.1.1a0" [tool.poetry.group.dev.dependencies] pytest = "^6.2.2" diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index 40cafad..e4fbbe2 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python" -version = "2023.5.2a0" +version = "2023.9.1" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 7e6595783f6ebf5c2bcce10670cf977644f89d27 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Fri, 5 Jan 2024 14:30:15 -0800 Subject: [PATCH 02/26] Update pyproject.toml --- gridappsd-field-bus-lib/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 9207b26..8b01ace 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -21,6 +21,10 @@ packages = [ { include = 'gridappsd'} ] +[tool.poetry.scripts] +# Add things in the form +# myscript = 'my_package:main' +context_manager = 'gridappsd.context_manager:main' [tool.poetry.dependencies] python = ">=3.8.1,<4.0" From 5760f3e11a7954a19499efbf6246e95a4848aef1 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Fri, 5 Jan 2024 14:58:19 -0800 Subject: [PATCH 03/26] added more request types for context manager --- .../field_interface/context_manager.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py index 8481bcb..b8d89c9 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py @@ -85,6 +85,16 @@ def on_request(self, message_bus, headers: Dict, message): reply_to = headers['reply-to'] message_bus.send(reply_to, self.registered_agents) + elif message['request_type'] == 'is_initialized': + reply_to = headers['reply-to'] + message = {'initialized':True} + message_bus.send(reply_to, message) + + elif message['request_type'] == 'control_command': + simulation_id = message['input']['simulation_id'] + self.ot_connection.send(t.simulation_input_topic(simulation_id), + message) + class SwitchAreaContextManager(SwitchAreaAgent): @@ -137,9 +147,14 @@ def on_request(self, message_bus, headers: Dict, message): elif message['request_type'] == 'is_initialized': reply_to = headers['reply-to'] - message = {'is_initialized':self.is_initialized} + message = {'initialized':True} message_bus.send(reply_to, message) + elif message['request_type'] == 'control_command': + simulation_id = message['input']['simulation_id'] + self.ot_connection.send(t.simulation_input_topic(simulation_id), + message) + class SecondaryAreaContextManager(SecondaryAreaAgent): @@ -189,6 +204,16 @@ def on_request(self, message_bus, headers: Dict, message): reply_to = headers['reply-to'] message_bus.send(reply_to, self.registered_agents) + elif message['request_type'] == 'is_initialized': + reply_to = headers['reply-to'] + message = {'initialized':True} + message_bus.send(reply_to, message) + + elif message['request_type'] == 'control_command': + simulation_id = message['input']['simulation_id'] + self.ot_connection.send(t.simulation_input_topic(simulation_id), + message) + def get_MessageBusDefinition(area_id: str) -> MessageBusDefinition: connection_args = {"GRIDAPPSD_ADDRESS": os.environ.get('GRIDAPPSD_ADDRESS',"tcp://gridappsd:61613"), From 446106b735ec931820423a22014af485a89ac73a Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Mon, 8 Jan 2024 15:11:13 -0800 Subject: [PATCH 04/26] added yapf formatting --- .../field_interface/agents/agents.py | 81 +++++++++++-------- gridappsd-field-bus-lib/pyproject.toml | 14 ++++ gridappsd-python-lib/pyproject.toml | 14 ++++ 3 files changed, 74 insertions(+), 35 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py index c6ab326..41206ad 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py @@ -17,6 +17,7 @@ from gridappsd.field_interface.gridappsd_field_bus import GridAPPSDMessageBus from gridappsd.field_interface.interfaces import (FieldMessageBus, MessageBusDefinition) + CIM_PROFILE = None IEC61970_301 = None @@ -25,7 +26,7 @@ _log = logging.getLogger(__name__) -def set_cim_profile(cim_profile:str, iec61970_301:int): +def set_cim_profile(cim_profile: str, iec61970_301: int): global CIM_PROFILE global IEC61970_301 global cim @@ -62,9 +63,10 @@ def __init__(self, self.simulation_id = simulation_id self.context = None - #TODO: Change params and connection to local connection - self.params = ConnectionParameters(cim_profile=CIM_PROFILE, iec61970_301=IEC61970_301) - + # TODO: Change params and connection to local connection + self.params = ConnectionParameters(cim_profile=CIM_PROFILE, + iec61970_301=IEC61970_301) + self.connection = GridappsdConnection(self.params) self.connection.cim_profile = cim_profile @@ -75,7 +77,7 @@ def __init__(self, if ('context_manager' not in self.app_id): self.agent_id = "da_" + self.app_id else: - self.agent_id = downstream_message_bus_def.id+'.context_manager' + self.agent_id = downstream_message_bus_def.id + '.context_manager' self.agent_area_dict = agent_area_dict @@ -95,7 +97,7 @@ def __init__(self, # self.context = ContextManager.get(self.feeder_id, self.area_id) - #if agent_dict is not None: + # if agent_dict is not None: # self.addressable_equipments = agent_dict['addressable_equipment'] # self.unaddressable_equipments = agent_dict['unaddressable_equipment'] @@ -111,7 +113,7 @@ def _connect(self): if ('context_manager' not in self.app_id): self.agent_id = "da_" + self.app_id + "_" + self.downstream_message_bus.id - + if self.agent_area_dict is None: context = LocalContext.get_context_by_message_bus( self.downstream_message_bus) @@ -124,7 +126,7 @@ def _connect(self): if ('context_manager' not in self.app_id): LocalContext.register_agent(self.downstream_message_bus, self.upstream_message_bus, self) - + def disconnect(self): if self.upstream_message_bus is not None: @@ -166,7 +168,6 @@ def subscribe_to_messages(self): self.app_id), self.on_upstream_message) - if ('context_manager' not in self.app_id): _log.debug( f"Subscribing to message on agents topics: \n {t.field_message_bus_agent_topic(self.downstream_message_bus.id, self.agent_id)} \ @@ -227,22 +228,26 @@ def get_registration_details(self): self.upstream_message_bus.id, self.downstream_message_bus.id) return dataclasses.asdict(details) - + def publish_downstream(self, message): - self.downstream_message_bus.send(t.field_message_bus_topic(self.downstream_message_bus.id), message) - - def publish_upstream(self, message): - self.upstream_message_bus.send(t.field_message_bus_topic(self.upstream_message_bus.id), message) + self.downstream_message_bus.send( + t.field_message_bus_topic(self.downstream_message_bus.id), message) + def publish_upstream(self, message): + self.upstream_message_bus.send( + t.field_message_bus_topic(self.upstream_message_bus.id), message) - def send_control_command(self, differenceBuilder : DifferenceBuilder): + def send_control_command(self, differenceBuilder: DifferenceBuilder): if self.simulation_id is not None: - LocalContext.send_control_command(self.downstream_message_bus, differenceBuilder) + LocalContext.send_control_command(self.downstream_message_bus, + differenceBuilder) + ''' TODO This block needs to be tested with device interface else: self.downstream_message_bus.send(devie_interface_topic, differenceBuilder) - ''' + ''' + ''' TODO this has not been implemented yet, so we are commented them out for now. # not all agent would use this @@ -261,20 +266,21 @@ def __init__(self, agent_config: Dict, feeder_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, - downstream_message_bus_def, agent_config, - feeder_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, feeder_dict, simulation_id) self.feeder_area = None self.downstream_message_bus_def = downstream_message_bus_def self._connect() if self.agent_area_dict is not None: - feeder = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + feeder = cim.EquipmentContainer( + mRID=self.downstream_message_bus_def.id) self.feeder_area = DistributedArea(connection=self.connection, - container=feeder, - distributed=True) - self.feeder_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) + container=feeder, + distributed=True) + self.feeder_area.build_from_topo_message( + topology_dict=self.agent_area_dict, centralized_graph=None) class SwitchAreaAgent(DistributedAgent): @@ -293,11 +299,13 @@ def __init__(self, self._connect() if self.agent_area_dict is not None: - container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + container = cim.EquipmentContainer( + mRID=self.downstream_message_bus_def.id) self.switch_area = DistributedArea(container=container, - connection=self.connection, - distributed=True) - self.switch_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) + connection=self.connection, + distributed=True) + self.switch_area.build_from_topo_message( + topology_dict=self.agent_area_dict, centralized_graph=None) class SecondaryAreaAgent(DistributedAgent): @@ -317,13 +325,16 @@ def __init__(self, if self.agent_area_dict is not None: if len(self.agent_area_dict['addressable_equipment']) == 0: - _log.warn(f"No addressable equipment in the secondary area with down stream message bus id: {self.downstream_message_bus.id}.") - container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) + _log.warning( + f"No addressable equipment in the secondary area with down stream message bus id: {self.downstream_message_bus.id}." + ) + container = cim.EquipmentContainer( + mRID=self.downstream_message_bus_def.id) self.secondary_area = DistributedArea(container=container, - connection=self.connection, - distributed=True) - self.secondary_area.build_from_topo_message(topology_dict=self.agent_area_dict, centralized_graph=None) - + connection=self.connection, + distributed=True) + self.secondary_area.build_from_topo_message( + topology_dict=self.agent_area_dict, centralized_graph=None) class CoordinatingAgent: @@ -347,7 +358,7 @@ def __init__(self, self.system_message_bus = GridAPPSDMessageBus(system_message_bus_def) self.system_message_bus.connect() - #This will change when we have multiple feeders per system + # This will change when we have multiple feeders per system self.downstream_message_bus = self.system_message_bus # self.context = ContextManager.getContextByFeeder(self.feeder_id) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 8b01ace..a135bdf 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -41,3 +41,17 @@ yapf = "^0.32.0" [build-system] requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api" + +[tool.yapfignore] +ignore_patterns = [ + ".venv/**", + ".pytest_cache/**", + "dist/**", + "docs/**" +] + +[tool.yapf] +based_on_style = "pep8" +spaces_before_comment = 4 +column_limit = 99 +split_before_logical_operator = true diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index e4fbbe2..67404f7 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -48,3 +48,17 @@ gitpython = "^3.1.31" [build-system] requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api" + +[tool.yapfignore] +ignore_patterns = [ + ".venv/**", + ".pytest_cache/**", + "dist/**", + "docs/**" +] + +[tool.yapf] +based_on_style = "pep8" +spaces_before_comment = 4 +column_limit = 99 +split_before_logical_operator = true From 90b802f77ba00028e89d4f1a5b90039fb9d79d9f Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Tue, 9 Jan 2024 10:49:37 -0800 Subject: [PATCH 05/26] Update package versions in pyproject.toml files --- gridappsd-field-bus-lib/pyproject.toml | 8 ++++---- gridappsd-python-lib/pyproject.toml | 2 +- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index a135bdf..15eee6a 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-field-bus" -version = "2024.1.1a0" +version = "2023.12.1" description = "GridAPPS-D Field Bus Implementation" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -28,8 +28,8 @@ context_manager = 'gridappsd.context_manager:main' [tool.poetry.dependencies] python = ">=3.8.1,<4.0" -gridappsd-python = {path="../gridappsd-python-lib", develop=true} -cim-graph = "^0.1.1a0" +gridappsd-python = ">2023" +cim-graph = ">=0.1.1a0" [tool.poetry.group.dev.dependencies] pytest = "^6.2.2" @@ -49,7 +49,7 @@ ignore_patterns = [ "dist/**", "docs/**" ] - + [tool.yapf] based_on_style = "pep8" spaces_before_comment = 4 diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index 67404f7..249e986 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python" -version = "2023.9.1" +version = "2023.12.1" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", diff --git a/pyproject.toml b/pyproject.toml index 4c6c427..29c98e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2023.9.1" +version = "2023.12.1" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 3481618e1ec00154e1893a6fef28ee005eaf0e55 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:51:33 -0800 Subject: [PATCH 06/26] Update deploy-dev-release.yml Remove most of bump_rules --- .github/workflows/deploy-dev-release.yml | 46 ++++++++++++------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 004972a..f6c398b 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -110,32 +110,32 @@ jobs: # we need to remove the v from the beginning of the tag. # echo "Bump rule is ${{ inputs.bump-rule }}" # echo "Given release version is ${{ inputs.release-version }}" - dt=$(date +%Y.%-m.0) - if ! $(git tag -l "v*" = ''); then - # uses -V which is version sort to keep it monotonically increasing. - current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) - echo "current git tag is ${current_tag}" - current_tag=${current_tag#?} - if [[ "$current_tag" < "$dt" ]]; then - current_tag=$dt - fi - # current_tag is now the version we want to set our poetry version so - # that we can bump the version - ./scripts/run_on_each.sh poetry version ${current_tag} - ./scripts/run_on_each.sh poetry version prerelease - - # poetry version ${current_tag} - # poetry version prerelease --no-interaction + # dt=$(date +%Y.%-m.0) + # if ! $(git tag -l "v*" = ''); then + # # uses -V which is version sort to keep it monotonically increasing. + # current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) + # echo "current git tag is ${current_tag}" + # current_tag=${current_tag#?} + # if [[ "$current_tag" < "$dt" ]]; then + # current_tag=$dt + # fi + # # current_tag is now the version we want to set our poetry version so + # # that we can bump the version + # ./scripts/run_on_each.sh poetry version ${current_tag} + # ./scripts/run_on_each.sh poetry version prerelease + + # # poetry version ${current_tag} + # # poetry version prerelease --no-interaction - else - # very first release. start with inputs.release-version + # else + # # very first release. start with inputs.release-version - echo "First release. Setting tag as 0.1.0rc0" - current_tag=$(date +%Y.%-m.1) - ./scripts/run_on_each.sh poetry version ${current_tag} + # echo "First release. Setting tag as 0.1.0rc0" + # current_tag=$(date +%Y.%-m.1) + # ./scripts/run_on_each.sh poetry version ${current_tag} - # poetry version ${current_tag} - fi + # # poetry version ${current_tag} + # fi NEW_TAG=v$(poetry version --short) From f35cdfa76bbad73c63d6d0bf38e5cd3cc3478fa8 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Thu, 11 Jan 2024 13:41:56 -0800 Subject: [PATCH 07/26] Update pyproject.toml --- gridappsd-field-bus-lib/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 15eee6a..e2e087f 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -24,7 +24,7 @@ packages = [ [tool.poetry.scripts] # Add things in the form # myscript = 'my_package:main' -context_manager = 'gridappsd.context_manager:main' +context_manager = 'gridappsd.field_interface.context_manager:_main' [tool.poetry.dependencies] python = ">=3.8.1,<4.0" From 420e4f1269fa13e00b7a59fa37f74165063dc61d Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Thu, 11 Jan 2024 13:49:57 -0800 Subject: [PATCH 08/26] Bump version of pyproject --- gridappsd-field-bus-lib/pyproject.toml | 2 +- gridappsd-python-lib/pyproject.toml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index e2e087f..789a43b 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-field-bus" -version = "2023.12.1" +version = "2023.12.2a0" description = "GridAPPS-D Field Bus Implementation" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index 249e986..23a861a 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python" -version = "2023.12.1" +version = "2023.12.2a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", diff --git a/pyproject.toml b/pyproject.toml index 29c98e2..af4141b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2023.12.1" +version = "2023.12.2a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 1a37b5d07ac7c08c57c109625abf4771aba9d00b Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:37:55 -0800 Subject: [PATCH 09/26] Update pyproject.toml Fix to use relative path rather than specific version. --- gridappsd-field-bus-lib/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 789a43b..1584d85 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -28,7 +28,7 @@ context_manager = 'gridappsd.field_interface.context_manager:_main' [tool.poetry.dependencies] python = ">=3.8.1,<4.0" -gridappsd-python = ">2023" +gridappsd-python = { path="../gridappsd-python-lib", develop = true} cim-graph = ">=0.1.1a0" [tool.poetry.group.dev.dependencies] From 6508e95de74e3cb45c0d956f3526ff9fafa49bdf Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:46:03 -0800 Subject: [PATCH 10/26] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index af4141b..f7026d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2023.12.2a0" +version = "2024.1.3a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 3aff25607b7b98b3ce82b768e8fd49e0451316b4 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:51:17 -0800 Subject: [PATCH 11/26] Update so that the outer poetry file runs the version --- .github/workflows/deploy-dev-release.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index f6c398b..8d8f730 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -126,16 +126,18 @@ jobs: # # poetry version ${current_tag} # # poetry version prerelease --no-interaction - + # else # # very first release. start with inputs.release-version - + # echo "First release. Setting tag as 0.1.0rc0" # current_tag=$(date +%Y.%-m.1) # ./scripts/run_on_each.sh poetry version ${current_tag} - + # # poetry version ${current_tag} # fi + version=$(poetry version --short) + ./scripts/run_on_each.sh poetry version ${version} NEW_TAG=v$(poetry version --short) @@ -175,11 +177,11 @@ jobs: # This is needed, because the poetry publish will fail at the top level of the project # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - cd gridappsd-python-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} poetry publish cd ../gridappsd-field-bus-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} poetry publish From dad016f1b050b3a96d0fb45e905f1ae4d287f9f8 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:54:48 -0800 Subject: [PATCH 12/26] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f7026d1..3eacaec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2024.1.3a0" +version = "2024.1.4a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 981e53a55670c6da335cbd0535e24fa6521191e7 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Tue, 20 Feb 2024 10:49:27 -0800 Subject: [PATCH 13/26] added time out except --- .../gridappsd/field_interface/context.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py index d29737f..3ebf1c0 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py @@ -3,8 +3,13 @@ import dataclasses import gridappsd.topics as t import json +import logging +from gridappsd.goss import TimeoutError +import time +_log = logging.getLogger(__name__) + class LocalContext: @@ -14,10 +19,16 @@ def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_m request = {'request_type' : 'get_context', 'modelId': feeder_mrid, 'areaId': area_id} - print(t.context_request_queue(downstream_message_bus.id)) - response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) - return response + response = None + while response is None: + try: + response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) + except TimeoutError: + _log.info("Context request timed out. Trying again...") + time.sleep(5) + return response + @classmethod def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): """ @@ -27,7 +38,15 @@ def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): request = {'request_type' : 'get_context', 'areaId': downstream_message_bus.id } - return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) + response = None + while response is None: + try: + response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) + except TimeoutError: + _log.info("Context request timed out. Trying again...") + time.sleep(5) + + return response @classmethod def register_agent(cls, downstream_message_bus: FieldMessageBus, upstream_message_bus: FieldMessageBus, agent): From 36c948206bd311305d36824b69e97f05782209ff Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Thu, 29 Feb 2024 14:39:28 -0800 Subject: [PATCH 14/26] Update context.py --- gridappsd-field-bus-lib/gridappsd/field_interface/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py index 3ebf1c0..ffa931a 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py @@ -24,7 +24,7 @@ def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_m try: response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) except TimeoutError: - _log.info("Context request timed out. Trying again...") + _log.warn("Context request timed out. Trying again...") time.sleep(5) return response @@ -43,7 +43,7 @@ def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): try: response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) except TimeoutError: - _log.info("Context request timed out. Trying again...") + _log.warn("Context request timed out. Trying again...") time.sleep(5) return response From 657b16b7a495f6ebbaa01eeba9dd30b0d5742d97 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Thu, 29 Feb 2024 14:40:03 -0800 Subject: [PATCH 15/26] Update context.py --- gridappsd-field-bus-lib/gridappsd/field_interface/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py index ffa931a..a0c2603 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py @@ -24,7 +24,7 @@ def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_m try: response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) except TimeoutError: - _log.warn("Context request timed out. Trying again...") + _log.warning("Context request timed out. Trying again...") time.sleep(5) return response @@ -43,7 +43,7 @@ def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): try: response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) except TimeoutError: - _log.warn("Context request timed out. Trying again...") + _log.warning("Context request timed out. Trying again...") time.sleep(5) return response From fd923fce9d96e4c124d3acea6cb8982d67b844be Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Fri, 29 Mar 2024 13:25:33 -0700 Subject: [PATCH 16/26] Update goss.py --- gridappsd-python-lib/gridappsd/goss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gridappsd-python-lib/gridappsd/goss.py b/gridappsd-python-lib/gridappsd/goss.py index 0151846..8178e4a 100644 --- a/gridappsd-python-lib/gridappsd/goss.py +++ b/gridappsd-python-lib/gridappsd/goss.py @@ -151,7 +151,7 @@ def send(self, topic, message): 'GOSS_SUBJECT': self.__token}) def get_response(self, topic, message, timeout=5): - id = datetime.now().strftime("%Y%m%d%h%M%S") + id = datetime.now().strftime("%Y%m%d%h%M%S%f")[:-3] reply_to = "/temp-queue/response.{}".format(id) if isinstance(message, str): From 908c9aa70d8f01c038cebea347f0bedd93af03a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 20:26:22 +0000 Subject: [PATCH 17/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/add-to-project.yml | 2 +- .github/workflows/deploy-dev-release.yml | 1 - .github/workflows/release.yml | 20 +- .gitignore | 1 - .pre-commit-config.yaml | 2 +- MANIFEST.in | 2 +- README.md | 17 +- VERSION | 2 +- docs/Makefile | 2 +- docs/source/conf.py | 27 +- docs/source/gridappsd.rst | 2 - .../gridappsd/field_interface/__init__.py | 5 +- .../field_interface/agents/__init__.py | 16 +- .../field_interface/agents/agents.py | 117 ++++---- .../gridappsd/field_interface/context.py | 70 ++--- .../field_interface/context_manager.py | 143 +++++----- .../field_interface/gridappsd_field_bus.py | 7 +- .../gridappsd/field_interface/interfaces.py | 30 +-- gridappsd-field-bus-lib/info/VERSION | 2 +- gridappsd-python-lib/README.md | 17 +- gridappsd-python-lib/conf/entrypoint.sh | 2 +- gridappsd-python-lib/gridappsd/__init__.py | 2 - gridappsd-python-lib/gridappsd/__main__.py | 8 +- .../gridappsd/app_registration.py | 20 +- .../gridappsd/difference_builder.py | 2 +- gridappsd-python-lib/gridappsd/goss.py | 55 ++-- gridappsd-python-lib/gridappsd/gridappsd.py | 59 ++--- gridappsd-python-lib/gridappsd/houses.py | 24 +- .../gridappsd/json_extension.py | 8 +- gridappsd-python-lib/gridappsd/loghandler.py | 11 +- .../gridappsd/register_app.py | 9 +- gridappsd-python-lib/gridappsd/simulation.py | 41 +-- gridappsd-python-lib/gridappsd/timeseries.py | 68 ++--- gridappsd-python-lib/gridappsd/topics.py | 40 +-- gridappsd-python-lib/gridappsd/utils.py | 6 +- gridappsd-python-lib/info/VERSION | 2 +- gridappsd-python-lib/pyproject.toml | 2 +- gridappsd-python-lib/tests/conftest.py | 13 +- gridappsd-python-lib/tests/run_gridappsd.py | 4 +- .../tests/test_docker_handler.py | 9 - gridappsd-python-lib/tests/test_goss.py | 8 - gridappsd-python-lib/tests/test_gridappsd.py | 250 +++++++++--------- gridappsd-python-lib/tests/test_houses.py | 2 - gridappsd-python-lib/tests/test_json.py | 11 +- gridappsd-python-lib/tests/test_logging.py | 8 - .../tests/test_logging_integration.py | 3 - gridappsd-python-lib/tests/test_simulation.py | 2 - info/VERSION | 2 +- listen_all_topics.py | 3 +- make_measurement_files.py | 12 +- run_simulation.py | 62 +++-- scripts/create_local_version.sh | 2 +- scripts/projects.sh | 2 +- scripts/replace_path_deps.sh | 2 +- 54 files changed, 583 insertions(+), 656 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index a566489..d3fa231 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -28,4 +28,4 @@ jobs: contains(github.event.pull_request.labels.*.name, 'bug') with: project: 'https://github.com/orgs/GRIDAPPSD/projects/7' - column_name: 'Labeled' \ No newline at end of file + column_name: 'Labeled' diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 8d8f730..65939b9 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -184,4 +184,3 @@ jobs: cd ../gridappsd-field-bus-lib poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} poetry publish - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dcdd084..cb7546c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ on: type: bool release-version: description: "Version number to use for this release, do not start with v." - required: true + required: true type: bool publish-to: description: "Publish to pypi or pypi-test" @@ -19,7 +19,7 @@ on: options: - "pypi" - "pypi-test" - + defaults: run: shell: bash @@ -117,12 +117,12 @@ jobs: # This is needed, because the poetry publish will fail at the top level of the project # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - cd gridappsd-python-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} poetry publish cd ../gridappsd-field-bus-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} poetry publish - name: Publish to pypi test @@ -132,16 +132,16 @@ jobs: set -x set -u set -e - + ./scripts/run_on_each.sh poetry config repositories.testpypi https://test.pypi.org/legacy/ # This is needed, because the poetry publish will fail at the top level of the project # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS='--repository testpypi'" >> $GITHUB_ENV - cd gridappsd-python-lib - poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} + cd gridappsd-python-lib + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish cd ../gridappsd-field-bus-lib - poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} - poetry publish \ No newline at end of file + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry publish diff --git a/.gitignore b/.gitignore index 751a919..e251546 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,3 @@ ENV/ # pytest cache .pytest_cache - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94c3e96..d748452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,4 +27,4 @@ repos: description: "A formatter for Python files." entry: yapf language: python - types: [ python ] \ No newline at end of file + types: [ python ] diff --git a/MANIFEST.in b/MANIFEST.in index c5c7c00..ce86e7c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include gridappsd/conf/* \ No newline at end of file +include gridappsd/conf/* diff --git a/README.md b/README.md index 635e419..018b7a4 100644 --- a/README.md +++ b/README.md @@ -178,9 +178,9 @@ integration tests available to be run. In order to execute these, you must have as above with dev-dependencies. During the testing phase the docker containers required for the tests are downloaded from -dockerhub and started. By default the `develop` tag is used to test the library using pytest. +dockerhub and started. By default the `develop` tag is used to test the library using pytest. One can customize the docker image tag by setting the environmental -variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing +variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing pytest with the following: ```shell script @@ -190,7 +190,7 @@ pytest with the following: export GRIDAPPSD_TAG_ENV=other_tag pytest -# Tests also require the username and password to be avaialable as environmental variables +# Tests also require the username and password to be avaialable as environmental variables # in order for them to properly run these tests export GRIDAPPSD_USER=user export GRIDAPPSD_PASSWORD=pass @@ -200,18 +200,18 @@ pytest ***NOTE: the first running the tests will download all of the docker images associated with the [GOSS-GridAPPS-D](http://github.com/GRIDAPPSD/GOSS-GridAPPS-D) repository. This process may take some time.*** - + ### Running tests created in a new project The `gridappsd-python` library exposes a testing environment through the `gridappsd.docker_handler` module. Including the following `conftest.py` in the root of your base test directory allows tests to reference these. Using these fixtures will start all of the -base containers required for `gridappsd` to run. +base containers required for `gridappsd` to run. ```python # conftest.py -# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. -# +# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. +# # Tested project structure an layout # # project-folder\ @@ -303,7 +303,7 @@ Using the above fixtures from inside a test module and test function looks like ```python -# Example test function using the gridappsd_client fixture +# Example test function using the gridappsd_client fixture @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) def test_gridappsd_status(gridappsd_client): @@ -325,4 +325,3 @@ def test_gridappsd_status(gridappsd_client): assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value ``` - diff --git a/VERSION b/VERSION index 17c91dc..dd41647 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2023.5.1 \ No newline at end of file +2023.5.1 diff --git a/docs/Makefile b/docs/Makefile index 99101bd..e411529 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/conf.py b/docs/source/conf.py index b50f164..29c4cd1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,10 +15,9 @@ import os import sys # insert top level of project to the path. -print("PATH IS: "+os.path.abspath("../../")) +print("PATH IS: " + os.path.abspath("../../")) sys.path.insert(0, os.path.abspath('../../')) - # -- Project information ----------------------------------------------------- project = u'GridAPPS-D Python' @@ -30,7 +29,6 @@ # The full version, including alpha/beta/rc tags release = u'1.0' - # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -80,7 +78,6 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -110,13 +107,11 @@ # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'GridAPPS-DPythondoc' - # -- Options for LaTeX output ------------------------------------------------ latex_elements = { @@ -145,16 +140,12 @@ u'Craig Allwardt', 'manual'), ] - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'gridapps-dpython', u'GridAPPS-D Python Documentation', - [author], 1) -] - +man_pages = [(master_doc, 'gridapps-dpython', + u'GridAPPS-D Python Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -167,7 +158,6 @@ 'Miscellaneous'), ] - # -- Options for Epub output ------------------------------------------------- # Bibliographic Dublin Core info. @@ -195,7 +185,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} - import subprocess @@ -208,17 +197,21 @@ def setup(app): :param app: """ app.connect('builder-inited', generate_apidoc) + + # app.connect('build-finished', clean_apirst) def generate_apidoc(app): - print('BUILIDING api docs '+ __file__) - print('CWD: '+os.getcwd()) + print('BUILIDING api docs ' + __file__) + print('CWD: ' + os.getcwd()) gridappsd_src = '../gridappsd' if os.environ.get("READTHEDOCS"): gridappsd_src = '../../gridappsd' - cmd = ["sphinx-apidoc", '-M', '-d 4', '-o', 'source', '--force', gridappsd_src] + cmd = [ + "sphinx-apidoc", '-M', '-d 4', '-o', 'source', '--force', gridappsd_src + ] subprocess.check_call(cmd) # print(path_to_src) diff --git a/docs/source/gridappsd.rst b/docs/source/gridappsd.rst index 452f0e7..d3bad45 100644 --- a/docs/source/gridappsd.rst +++ b/docs/source/gridappsd.rst @@ -40,5 +40,3 @@ gridappsd.topics module :members: :undoc-members: :show-inheritance: - - diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py index 66eba6b..3385c5f 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py @@ -3,7 +3,4 @@ from gridappsd.field_interface.context import LocalContext from gridappsd.field_interface.interfaces import MessageBusDefinition -__all__: List[str] = [ - "LocalContext", - "MessageBusDefinition" -] +__all__: List[str] = ["LocalContext", "MessageBusDefinition"] diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py index d972d84..92f6359 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py @@ -1,15 +1,7 @@ from typing import List -from gridappsd.field_interface.agents.agents import ( - FeederAgent, - DistributedAgent, - CoordinatingAgent, - SwitchAreaAgent, - SecondaryAreaAgent -) +from gridappsd.field_interface.agents.agents import (FeederAgent, DistributedAgent, + CoordinatingAgent, SwitchAreaAgent, + SecondaryAreaAgent) -__all__: List[str] = [ - "FeederAgent", - "DistributedAgent", - "CoordinatingAgent" -] +__all__: List[str] = ["FeederAgent", "DistributedAgent", "CoordinatingAgent"] diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py index 41206ad..01a7aaf 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py @@ -15,9 +15,7 @@ import gridappsd.topics as t from gridappsd.field_interface.context import LocalContext from gridappsd.field_interface.gridappsd_field_bus import GridAPPSDMessageBus -from gridappsd.field_interface.interfaces import (FieldMessageBus, - MessageBusDefinition) - +from gridappsd.field_interface.interfaces import (FieldMessageBus, MessageBusDefinition) CIM_PROFILE = None IEC61970_301 = None @@ -64,8 +62,7 @@ def __init__(self, self.context = None # TODO: Change params and connection to local connection - self.params = ConnectionParameters(cim_profile=CIM_PROFILE, - iec61970_301=IEC61970_301) + self.params = ConnectionParameters(cim_profile=CIM_PROFILE, iec61970_301=IEC61970_301) self.connection = GridappsdConnection(self.params) self.connection.cim_profile = cim_profile @@ -83,15 +80,13 @@ def __init__(self, if upstream_message_bus_def is not None: if upstream_message_bus_def.is_ot_bus: - self.upstream_message_bus = GridAPPSDMessageBus( - upstream_message_bus_def) + self.upstream_message_bus = GridAPPSDMessageBus(upstream_message_bus_def) # else: # self.upstream_message_bus = VolttronMessageBus(upstream_message_bus_def) if downstream_message_bus_def is not None: if downstream_message_bus_def.is_ot_bus: - self.downstream_message_bus = GridAPPSDMessageBus( - downstream_message_bus_def) + self.downstream_message_bus = GridAPPSDMessageBus(downstream_message_bus_def) # else: # self.downstream_message_bus = VolttronMessageBus(downstream_message_bus_def) @@ -108,15 +103,13 @@ def _connect(self): if self.downstream_message_bus is not None: self.downstream_message_bus.connect() if self.downstream_message_bus is None and self.upstream_message_bus is None: - raise ValueError( - "Either upstream or downstream bus must be specified!") + raise ValueError("Either upstream or downstream bus must be specified!") if ('context_manager' not in self.app_id): self.agent_id = "da_" + self.app_id + "_" + self.downstream_message_bus.id if self.agent_area_dict is None: - context = LocalContext.get_context_by_message_bus( - self.downstream_message_bus) + context = LocalContext.get_context_by_message_bus(self.downstream_message_bus) self.agent_area_dict = context['data'] self.subscribe_to_measurement() @@ -124,8 +117,8 @@ def _connect(self): self.subscribe_to_requests() if ('context_manager' not in self.app_id): - LocalContext.register_agent(self.downstream_message_bus, - self.upstream_message_bus, self) + LocalContext.register_agent(self.downstream_message_bus, self.upstream_message_bus, + self) def disconnect(self): @@ -137,35 +130,28 @@ def disconnect(self): def subscribe_to_measurement(self): if self.simulation_id is None: self.downstream_message_bus.subscribe( - t.field_output_topic(self.downstream_message_bus.id), - self.on_measurement) + t.field_output_topic(self.downstream_message_bus.id), self.on_measurement) else: - topic = t.field_output_topic(self.downstream_message_bus.id, - self.simulation_id) + topic = t.field_output_topic(self.downstream_message_bus.id, self.simulation_id) _log.debug(f"subscribing to simulation output on topic {topic}") - self.downstream_message_bus.subscribe(topic, - self.on_simulation_output) + self.downstream_message_bus.subscribe(topic, self.on_simulation_output) def subscribe_to_messages(self): self.downstream_message_bus.subscribe( - t.field_message_bus_topic(self.downstream_message_bus.id), - self.on_downstream_message) + t.field_message_bus_topic(self.downstream_message_bus.id), self.on_downstream_message) self.upstream_message_bus.subscribe( - t.field_message_bus_topic(self.upstream_message_bus.id), - self.on_upstream_message) + t.field_message_bus_topic(self.upstream_message_bus.id), self.on_upstream_message) _log.debug( f"Subscribing to messages on application topics: \n {t.field_message_bus_app_topic(self.downstream_message_bus.id, self.app_id)} \ \n {t.field_message_bus_app_topic(self.upstream_message_bus.id, self.app_id)}" ) self.downstream_message_bus.subscribe( - t.field_message_bus_app_topic(self.downstream_message_bus.id, - self.app_id), + t.field_message_bus_app_topic(self.downstream_message_bus.id, self.app_id), self.on_downstream_message) self.upstream_message_bus.subscribe( - t.field_message_bus_app_topic(self.upstream_message_bus.id, - self.app_id), + t.field_message_bus_app_topic(self.upstream_message_bus.id, self.app_id), self.on_upstream_message) if ('context_manager' not in self.app_id): @@ -174,12 +160,10 @@ def subscribe_to_messages(self): \n {t.field_message_bus_agent_topic(self.upstream_message_bus.id, self.agent_id)}" ) self.downstream_message_bus.subscribe( - t.field_message_bus_agent_topic(self.downstream_message_bus.id, - self.agent_id), + t.field_message_bus_agent_topic(self.downstream_message_bus.id, self.agent_id), self.on_downstream_message) self.upstream_message_bus.subscribe( - t.field_message_bus_agent_topic(self.upstream_message_bus.id, - self.agent_id), + t.field_message_bus_agent_topic(self.upstream_message_bus.id, self.agent_id), self.on_upstream_message) def subscribe_to_requests(self): @@ -189,28 +173,23 @@ def subscribe_to_requests(self): \n {t.field_agent_request_queue(self.upstream_message_bus.id, self.agent_id)}" ) self.downstream_message_bus.subscribe( - t.field_agent_request_queue(self.downstream_message_bus.id, - self.agent_id), + t.field_agent_request_queue(self.downstream_message_bus.id, self.agent_id), self.on_request_from_downstream) self.upstream_message_bus.subscribe( - t.field_agent_request_queue(self.upstream_message_bus.id, - self.agent_id), + t.field_agent_request_queue(self.upstream_message_bus.id, self.agent_id), self.on_request_from_uptream) def on_measurement(self, headers: Dict, message) -> None: - raise NotImplementedError( - f"{self.__class__.__name__} must be overriden in child class") + raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") def on_simulation_output(self, headers, message): self.on_measurement(headers=headers, message=message) def on_upstream_message(self, headers: Dict, message) -> None: - raise NotImplementedError( - f"{self.__class__.__name__} must be overriden in child class") + raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") def on_downstream_message(self, headers: Dict, message) -> None: - raise NotImplementedError( - f"{self.__class__.__name__} must be overriden in child class") + raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") def on_request_from_uptream(self, headers: Dict, message): self.on_request(self.upstream_message_bus, headers, message) @@ -219,28 +198,25 @@ def on_request_from_downstream(self, headers: Dict, message): self.on_request(self.downstream_message_bus, headers, message) def on_request(self, message_bus, headers: Dict, message): - raise NotImplementedError( - f"{self.__class__.__name__} must be overriden in child class") + raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") def get_registration_details(self): - details = AgentRegistrationDetails(str(self.agent_id), self.app_id, - self.description, + details = AgentRegistrationDetails(str(self.agent_id), self.app_id, self.description, self.upstream_message_bus.id, self.downstream_message_bus.id) return dataclasses.asdict(details) def publish_downstream(self, message): - self.downstream_message_bus.send( - t.field_message_bus_topic(self.downstream_message_bus.id), message) + self.downstream_message_bus.send(t.field_message_bus_topic(self.downstream_message_bus.id), + message) def publish_upstream(self, message): - self.upstream_message_bus.send( - t.field_message_bus_topic(self.upstream_message_bus.id), message) + self.upstream_message_bus.send(t.field_message_bus_topic(self.upstream_message_bus.id), + message) def send_control_command(self, differenceBuilder: DifferenceBuilder): if self.simulation_id is not None: - LocalContext.send_control_command(self.downstream_message_bus, - differenceBuilder) + LocalContext.send_control_command(self.downstream_message_bus, differenceBuilder) ''' TODO This block needs to be tested with device interface @@ -250,7 +226,7 @@ def send_control_command(self, differenceBuilder: DifferenceBuilder): ''' TODO this has not been implemented yet, so we are commented them out for now. - # not all agent would use this + # not all agent would use this def on_control(self, control): device_id = control.get('device') command = control.get('command') @@ -266,21 +242,20 @@ def __init__(self, agent_config: Dict, feeder_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, feeder_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + feeder_dict, simulation_id) self.feeder_area = None self.downstream_message_bus_def = downstream_message_bus_def self._connect() if self.agent_area_dict is not None: - feeder = cim.EquipmentContainer( - mRID=self.downstream_message_bus_def.id) + feeder = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) self.feeder_area = DistributedArea(connection=self.connection, container=feeder, distributed=True) - self.feeder_area.build_from_topo_message( - topology_dict=self.agent_area_dict, centralized_graph=None) + self.feeder_area.build_from_topo_message(topology_dict=self.agent_area_dict, + centralized_graph=None) class SwitchAreaAgent(DistributedAgent): @@ -291,21 +266,20 @@ def __init__(self, agent_config: Dict, switch_area_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, switch_area_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + switch_area_dict, simulation_id) self.switch_area = None self.downstream_message_bus_def = downstream_message_bus_def self._connect() if self.agent_area_dict is not None: - container = cim.EquipmentContainer( - mRID=self.downstream_message_bus_def.id) + container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) self.switch_area = DistributedArea(container=container, connection=self.connection, distributed=True) - self.switch_area.build_from_topo_message( - topology_dict=self.agent_area_dict, centralized_graph=None) + self.switch_area.build_from_topo_message(topology_dict=self.agent_area_dict, + centralized_graph=None) class SecondaryAreaAgent(DistributedAgent): @@ -316,8 +290,8 @@ def __init__(self, agent_config: Dict, secondary_area_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, secondary_area_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + secondary_area_dict, simulation_id) self.secondary_area = None self.downstream_message_bus_def = downstream_message_bus_def @@ -328,13 +302,12 @@ def __init__(self, _log.warning( f"No addressable equipment in the secondary area with down stream message bus id: {self.downstream_message_bus.id}." ) - container = cim.EquipmentContainer( - mRID=self.downstream_message_bus_def.id) + container = cim.EquipmentContainer(mRID=self.downstream_message_bus_def.id) self.secondary_area = DistributedArea(container=container, connection=self.connection, distributed=True) - self.secondary_area.build_from_topo_message( - topology_dict=self.agent_area_dict, centralized_graph=None) + self.secondary_area.build_from_topo_message(topology_dict=self.agent_area_dict, + centralized_graph=None) class CoordinatingAgent: diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py index a0c2603..6527730 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py @@ -7,77 +7,85 @@ from gridappsd.goss import TimeoutError import time - _log = logging.getLogger(__name__) class LocalContext: - + @classmethod - def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_mrid, area_id=None): - - request = {'request_type' : 'get_context', - 'modelId': feeder_mrid, - 'areaId': area_id} + def get_context_by_feeder(cls, + downstream_message_bus: FieldMessageBus, + feeder_mrid, + area_id=None): + + request = {'request_type': 'get_context', 'modelId': feeder_mrid, 'areaId': area_id} response = None while response is None: - try: - response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) - except TimeoutError: - _log.warning("Context request timed out. Trying again...") - time.sleep(5) + try: + response = downstream_message_bus.get_response(t.context_request_queue( + downstream_message_bus.id), + request, + timeout=10) + except TimeoutError: + _log.warning("Context request timed out. Trying again...") + time.sleep(5) return response - + @classmethod def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): """ return agents/devices based on downstream message bus as input """ - request = {'request_type' : 'get_context', - 'areaId': downstream_message_bus.id - } + request = {'request_type': 'get_context', 'areaId': downstream_message_bus.id} response = None while response is None: try: - response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) + response = downstream_message_bus.get_response(t.context_request_queue( + downstream_message_bus.id), + request, + timeout=10) except TimeoutError: _log.warning("Context request timed out. Trying again...") time.sleep(5) - + return response - + @classmethod - def register_agent(cls, downstream_message_bus: FieldMessageBus, upstream_message_bus: FieldMessageBus, agent): + def register_agent(cls, downstream_message_bus: FieldMessageBus, + upstream_message_bus: FieldMessageBus, agent): """ Sends the newly created distributed agent's info to OT bus """ - request = {'request_type' : 'register_agent', - 'agent' : agent.get_registration_details()} + request = {'request_type': 'register_agent', 'agent': agent.get_registration_details()} downstream_message_bus.send(t.context_request_queue(downstream_message_bus.id), request) upstream_message_bus.send(t.context_request_queue(upstream_message_bus.id), request) - + @classmethod def get_agents(cls, downstream_message_bus: FieldMessageBus): """ Sends the newly created distributed agent's info to OT bus """ - request = {'request_type' : 'get_agents'} - return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) - + request = {'request_type': 'get_agents'} + return downstream_message_bus.get_response( + t.context_request_queue(downstream_message_bus.id), request) + @classmethod - def send_control_command(cls, downstream_message_bus: FieldMessageBus, difference_builder: DifferenceBuilder): + def send_control_command(cls, downstream_message_bus: FieldMessageBus, + difference_builder: DifferenceBuilder): """ Sends the control command to device """ - request = {'request_type' : 'control_command', - 'difference_builder': difference_builder.get_message()} - downstream_message_bus.send(t.context_request_queue(downstream_message_bus.id), - request) + request = { + 'request_type': 'control_command', + 'difference_builder': difference_builder.get_message() + } + downstream_message_bus.send(t.context_request_queue(downstream_message_bus.id), request) + # Provide context based on router (ip trace) or PKI # Maybe able to emulate/simulate diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py index b8d89c9..8b97060 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py @@ -8,14 +8,12 @@ import gridappsd.topics as t from cimgraph.data_profile import CIM_PROFILE from gridappsd import GridAPPSD -from gridappsd.field_interface.agents import (FeederAgent, SecondaryAreaAgent, - SwitchAreaAgent) +from gridappsd.field_interface.agents import (FeederAgent, SecondaryAreaAgent, SwitchAreaAgent) from gridappsd.field_interface.interfaces import MessageBusDefinition - cim_profile = CIM_PROFILE.RC4_2021.value -agents_mod.set_cim_profile(cim_profile=cim_profile,iec61970_301=7) +agents_mod.set_cim_profile(cim_profile=cim_profile, iec61970_301=7) cim = agents_mod.cim @@ -40,23 +38,23 @@ def __init__(self, self.ot_connection = GridAPPSD() if feeder_dict is None: - request = {'request_type':'get_context', - 'modelId': downstream_message_bus_def.id} + request = {'request_type': 'get_context', 'modelId': downstream_message_bus_def.id} feeder_dict = None while feeder_dict is None: self.ot_connection.get_logger().info(f"Requesting topology for {self.__class__}") response = self.ot_connection.get_response(REQUEST_FIELD, request, timeout=10) if 'data' in response: feeder_dict = response['data'] - self.ot_connection.send_status("******RCVD FEEDER *************", '/topic/goss.gridappsd.platform.log', 'DEBUG') + self.ot_connection.send_status("******RCVD FEEDER *************", + '/topic/goss.gridappsd.platform.log', 'DEBUG') else: time.sleep(5) - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, feeder_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + feeder_dict, simulation_id) #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - + self.context = None self.registered_agents = {} @@ -74,12 +72,11 @@ def on_request(self, message_bus, headers: Dict, message): reply_to = headers['reply-to'] if self.context is None: self.context = self.ot_connection.get_response(REQUEST_FIELD, message) - message_bus.send(reply_to,self.context) + message_bus.send(reply_to, self.context) elif message['request_type'] == 'register_agent': self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) - self.registered_agents[message['agent'] - ['agent_id']] = message['agent'] + self.registered_agents[message['agent']['agent_id']] = message['agent'] elif message['request_type'] == 'get_agents': reply_to = headers['reply-to'] @@ -87,13 +84,12 @@ def on_request(self, message_bus, headers: Dict, message): elif message['request_type'] == 'is_initialized': reply_to = headers['reply-to'] - message = {'initialized':True} + message = {'initialized': True} message_bus.send(reply_to, message) elif message['request_type'] == 'control_command': simulation_id = message['input']['simulation_id'] - self.ot_connection.send(t.simulation_input_topic(simulation_id), - message) + self.ot_connection.send(t.simulation_input_topic(simulation_id), message) class SwitchAreaContextManager(SwitchAreaAgent): @@ -107,17 +103,16 @@ def __init__(self, self.ot_connection = GridAPPSD() if switch_area_dict is None: - request = {'request_type':'get_context', - 'areaId': downstream_message_bus_def.id} - switch_area_dict = self.ot_connection.get_response( - REQUEST_FIELD, request, timeout=10)['data'] + request = {'request_type': 'get_context', 'areaId': downstream_message_bus_def.id} + switch_area_dict = self.ot_connection.get_response(REQUEST_FIELD, request, + timeout=10)['data'] - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, switch_area_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + switch_area_dict, simulation_id) #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - + self.context = None self.registered_agents = {} @@ -132,28 +127,26 @@ def on_request(self, message_bus, headers: Dict, message): reply_to = headers['reply-to'] if self.context is None: self.context = self.ot_connection.get_response(REQUEST_FIELD, message) - message_bus.send(reply_to,self.context) + message_bus.send(reply_to, self.context) elif message['request_type'] == 'register_agent': #TODO: check for initialization self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) - self.registered_agents[message['agent'] - ['agent_id']] = message['agent'] + self.registered_agents[message['agent']['agent_id']] = message['agent'] elif message['request_type'] == 'get_agents': #TODO: check for initialization reply_to = headers['reply-to'] message_bus.send(reply_to, self.registered_agents) - + elif message['request_type'] == 'is_initialized': reply_to = headers['reply-to'] - message = {'initialized':True} + message = {'initialized': True} message_bus.send(reply_to, message) elif message['request_type'] == 'control_command': simulation_id = message['input']['simulation_id'] - self.ot_connection.send(t.simulation_input_topic(simulation_id), - message) + self.ot_connection.send(t.simulation_input_topic(simulation_id), message) class SecondaryAreaContextManager(SecondaryAreaAgent): @@ -167,17 +160,17 @@ def __init__(self, self.ot_connection = GridAPPSD() if secondary_area_dict is None: - request = {'request_type':'get_context', - 'areaId': downstream_message_bus_def.id} - secondary_area_dict = self.ot_connection.get_response( - REQUEST_FIELD, request, timeout=10)['data'] + request = {'request_type': 'get_context', 'areaId': downstream_message_bus_def.id} + secondary_area_dict = self.ot_connection.get_response(REQUEST_FIELD, + request, + timeout=10)['data'] - super().__init__(upstream_message_bus_def, downstream_message_bus_def, - agent_config, secondary_area_dict, simulation_id) + super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, + secondary_area_dict, simulation_id) #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - + self.context = None self.registered_agents = {} @@ -192,13 +185,11 @@ def on_request(self, message_bus, headers: Dict, message): reply_to = headers['reply-to'] if self.context is None: self.context = self.ot_connection.get_response(REQUEST_FIELD, message) - message_bus.send(reply_to,self.context) - + message_bus.send(reply_to, self.context) elif message['request_type'] == 'register_agent': self.ot_connection.send(t.REGISTER_AGENT_QUEUE, message) - self.registered_agents[message['agent'] - ['agent_id']] = message['agent'] + self.registered_agents[message['agent']['agent_id']] = message['agent'] elif message['request_type'] == 'get_agents': reply_to = headers['reply-to'] @@ -206,36 +197,38 @@ def on_request(self, message_bus, headers: Dict, message): elif message['request_type'] == 'is_initialized': reply_to = headers['reply-to'] - message = {'initialized':True} + message = {'initialized': True} message_bus.send(reply_to, message) elif message['request_type'] == 'control_command': simulation_id = message['input']['simulation_id'] - self.ot_connection.send(t.simulation_input_topic(simulation_id), - message) + self.ot_connection.send(t.simulation_input_topic(simulation_id), message) + def get_MessageBusDefinition(area_id: str) -> MessageBusDefinition: - - connection_args = {"GRIDAPPSD_ADDRESS": os.environ.get('GRIDAPPSD_ADDRESS',"tcp://gridappsd:61613"), - "GRIDAPPSD_USER": os.environ.get('GRIDAPPSD_USER'), - "GRIDAPPSD_PASSWORD": os.environ.get('GRIDAPPSD_PASSWORD'), - "GRIDAPPSD_APPLICATION_ID": os.environ.get('GRIDAPPSD_APPLICATION_ID')} - - bus = MessageBusDefinition(id=area_id, - is_ot_bus= True, - connection_type = "GRIDAPPSD_TYPE_GRIDAPPSD", - conneciton_args = connection_args) - + + connection_args = { + "GRIDAPPSD_ADDRESS": os.environ.get('GRIDAPPSD_ADDRESS', "tcp://gridappsd:61613"), + "GRIDAPPSD_USER": os.environ.get('GRIDAPPSD_USER'), + "GRIDAPPSD_PASSWORD": os.environ.get('GRIDAPPSD_PASSWORD'), + "GRIDAPPSD_APPLICATION_ID": os.environ.get('GRIDAPPSD_APPLICATION_ID') + } + + bus = MessageBusDefinition(id=area_id, + is_ot_bus=True, + connection_type="GRIDAPPSD_TYPE_GRIDAPPSD", + conneciton_args=connection_args) + return bus + def _main(): time.sleep(10) parser = argparse.ArgumentParser() parser.add_argument( "--simulation_id", - help= - "Simulation id to use for communicating with simulated devices on the message bus. \ + help="Simulation id to use for communicating with simulated devices on the message bus. \ If simulation_id is not provided then Context Manager assumes to run on deployed field with real devices.", required=False) opts = parser.parse_args() @@ -248,15 +241,14 @@ def _main(): "This agent provides topological context information like neighboring agents and devices to other distributed agents" } - gapps = GridAPPSD() - response = gapps.get_response(t.PLATFORM_STATUS, {"isField":True}) + response = gapps.get_response(t.PLATFORM_STATUS, {"isField": True}) field_model_mrid = response['fieldModelMrid'] is_field_initialized = False while not is_field_initialized: - response = gapps.get_response(REQUEST_FIELD, {"request_type":"is_initilized"}) + response = gapps.get_response(REQUEST_FIELD, {"request_type": "is_initilized"}) print(response) is_field_initialized = response['data']['initialized'] time.sleep(1) @@ -270,28 +262,23 @@ def _main(): agent_config, simulation_id=simulation_id) - - for switch_area in feeder_agent.agent_area_dict['switch_areas']: switch_area_message_bus_def = get_MessageBusDefinition(str(switch_area['message_bus_id'])) - print("Creating switch area agent " + - str(switch_area['message_bus_id'])) - switch_area_agent = SwitchAreaContextManager( - feeder_message_bus_def, - switch_area_message_bus_def, - agent_config, - simulation_id=simulation_id) - + print("Creating switch area agent " + str(switch_area['message_bus_id'])) + switch_area_agent = SwitchAreaContextManager(feeder_message_bus_def, + switch_area_message_bus_def, + agent_config, + simulation_id=simulation_id) + # create secondary area distributed agents for secondary_area in switch_area['secondary_areas']: - secondary_area_message_bus_def = get_MessageBusDefinition(str(secondary_area['message_bus_id'])) - print("Creating secondary area agent " + - str(secondary_area['message_bus_id'])) - secondary_area_agent = SecondaryAreaContextManager( - switch_area_message_bus_def, - secondary_area_message_bus_def, - agent_config, - simulation_id=simulation_id) + secondary_area_message_bus_def = get_MessageBusDefinition( + str(secondary_area['message_bus_id'])) + print("Creating secondary area agent " + str(secondary_area['message_bus_id'])) + secondary_area_agent = SecondaryAreaContextManager(switch_area_message_bus_def, + secondary_area_message_bus_def, + agent_config, + simulation_id=simulation_id) while True: try: diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py b/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py index ea9cfda..43c9963 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py @@ -5,6 +5,7 @@ class GridAPPSDMessageBus(FieldMessageBus): + def __init__(self, definition: MessageBusDefinition): super(GridAPPSDMessageBus, self).__init__(definition) self._id = definition.id @@ -14,7 +15,7 @@ def __init__(self, definition: MessageBusDefinition): self._address = definition.conneciton_args["GRIDAPPSD_ADDRESS"] self.gridappsd_obj = None - + def query_devices(self) -> dict: pass @@ -43,14 +44,14 @@ def send(self, topic: str, message: Any): """ if self.gridappsd_obj is not None: self.gridappsd_obj.send(topic, message) - + def get_response(self, topic, message, timeout=5): """ Sends a message on a specific concrete queue, waits and returns the response """ if self.gridappsd_obj is not None: return self.gridappsd_obj.get_response(topic, message, timeout) - + def disconnect(self): """ Disconnect from the concrete message bus. diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py b/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py index 8e42e66..237bac3 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py @@ -9,10 +9,8 @@ from pathlib import Path from typing import Dict, List, Optional, Union - import yaml - _log = logging.getLogger(__name__) @@ -37,6 +35,7 @@ class ConnectionType(Enum): class ProtocolTransformer(ABC): + @staticmethod @abstractmethod def to_cim(data) -> str: @@ -58,34 +57,29 @@ def to_protocol(cim_data: str, from_format: Optional[str] = None): and return it. cim_data: string representing cim data structures/change structure - from_format: specifies the type + from_format: specifies the type """ pass - @dataclass class MessageBusDefinition: """ A `MessageBusDefinition` class is used to define how to connect to the message bus. """ - """ A global unique string representing a specific message bus. """ id: str - """ connection_type describes how the agent/endpoint will connect to the message bus """ connection_type: ConnectionType - """ connection_args allows dynamic key/value paired strings to be added to allow connections. """ conneciton_args: Dict[str, str] - """ Determines whether or not this message bus has the role of ot bus. """ @@ -96,14 +90,15 @@ def load(config_file) -> MessageBusDefinition: """ """ - config = yaml.load(open(config_file),Loader=yaml.FullLoader)['connections'] - + config = yaml.load(open(config_file), Loader=yaml.FullLoader)['connections'] + required = ["id", "connection_type", "connection_args"] for k in required: if k not in config: raise ValueError(f"Missing keys for connection {k}") - definition = MessageBusDefinition(config[required[0]], config[required[1]], config[required[2]]) + definition = MessageBusDefinition(config[required[0]], config[required[1]], + config[required[2]]) for k in config: if k == "connection_args": definition.conneciton_args = dict() @@ -114,11 +109,12 @@ def load(config_file) -> MessageBusDefinition: if not hasattr(definition, "is_ot_bus"): setattr(definition, "is_ot_bus", False) - + return definition class FieldMessageBus: + def __init__(self, config: MessageBusDefinition): self._devices = dict() self._is_ot_bus = config.is_ot_bus @@ -170,18 +166,18 @@ def send(self, topic, message): Publish device specific message to the concrete message bus. """ pass - + @abstractmethod def get_response(self, topic, message, timeout): """ Sends a message on a specific queue, waits and returns the response """ - + def get_agent_response(self, agent_id, message, timeout): """ Sends a message on a specific agent's request queue, waits and returns the response """ - topic = "{}.request.{}.{}".format(t.BASE_FIELD_QUEUE,self.id, agent_id) + topic = "{}.request.{}.{}".format(t.BASE_FIELD_QUEUE, self.id, agent_id) self.get_response(topic, message, timeout) @abstractmethod @@ -193,6 +189,7 @@ def disconnect(self): class MessageBusDefinitions: + def __init__( self, config: Optional[Union[dict, str]] = None, @@ -245,6 +242,7 @@ def __next__(self) -> MessageBusDefinition: class DeviceFieldInterface: + def __init__( self, id: str, @@ -281,6 +279,6 @@ def on_control_message(self, message): def on_state_change(self): """ This event should be triggered by the device/protocol that - is used. + is used. """ pass diff --git a/gridappsd-field-bus-lib/info/VERSION b/gridappsd-field-bus-lib/info/VERSION index 17c91dc..dd41647 100644 --- a/gridappsd-field-bus-lib/info/VERSION +++ b/gridappsd-field-bus-lib/info/VERSION @@ -1 +1 @@ -2023.5.1 \ No newline at end of file +2023.5.1 diff --git a/gridappsd-python-lib/README.md b/gridappsd-python-lib/README.md index 635e419..018b7a4 100644 --- a/gridappsd-python-lib/README.md +++ b/gridappsd-python-lib/README.md @@ -178,9 +178,9 @@ integration tests available to be run. In order to execute these, you must have as above with dev-dependencies. During the testing phase the docker containers required for the tests are downloaded from -dockerhub and started. By default the `develop` tag is used to test the library using pytest. +dockerhub and started. By default the `develop` tag is used to test the library using pytest. One can customize the docker image tag by setting the environmental -variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing +variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing pytest with the following: ```shell script @@ -190,7 +190,7 @@ pytest with the following: export GRIDAPPSD_TAG_ENV=other_tag pytest -# Tests also require the username and password to be avaialable as environmental variables +# Tests also require the username and password to be avaialable as environmental variables # in order for them to properly run these tests export GRIDAPPSD_USER=user export GRIDAPPSD_PASSWORD=pass @@ -200,18 +200,18 @@ pytest ***NOTE: the first running the tests will download all of the docker images associated with the [GOSS-GridAPPS-D](http://github.com/GRIDAPPSD/GOSS-GridAPPS-D) repository. This process may take some time.*** - + ### Running tests created in a new project The `gridappsd-python` library exposes a testing environment through the `gridappsd.docker_handler` module. Including the following `conftest.py` in the root of your base test directory allows tests to reference these. Using these fixtures will start all of the -base containers required for `gridappsd` to run. +base containers required for `gridappsd` to run. ```python # conftest.py -# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. -# +# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. +# # Tested project structure an layout # # project-folder\ @@ -303,7 +303,7 @@ Using the above fixtures from inside a test module and test function looks like ```python -# Example test function using the gridappsd_client fixture +# Example test function using the gridappsd_client fixture @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) def test_gridappsd_status(gridappsd_client): @@ -325,4 +325,3 @@ def test_gridappsd_status(gridappsd_client): assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value ``` - diff --git a/gridappsd-python-lib/conf/entrypoint.sh b/gridappsd-python-lib/conf/entrypoint.sh index 53e7107..ceb36d4 100755 --- a/gridappsd-python-lib/conf/entrypoint.sh +++ b/gridappsd-python-lib/conf/entrypoint.sh @@ -15,4 +15,4 @@ done echo "[Entrypoint] Starting the platform" /startup/conf/run-gridappsd.sh -tail -f /dev/null \ No newline at end of file +tail -f /dev/null diff --git a/gridappsd-python-lib/gridappsd/__init__.py b/gridappsd-python-lib/gridappsd/__init__.py index e57c6ab..3a36d69 100644 --- a/gridappsd-python-lib/gridappsd/__init__.py +++ b/gridappsd-python-lib/gridappsd/__init__.py @@ -54,5 +54,3 @@ from gridappsd.difference_builder import DifferenceBuilder from gridappsd.app_registration import ApplicationController import gridappsd.json_extension as json - - diff --git a/gridappsd-python-lib/gridappsd/__main__.py b/gridappsd-python-lib/gridappsd/__main__.py index ade4e79..9a3ed57 100644 --- a/gridappsd-python-lib/gridappsd/__main__.py +++ b/gridappsd-python-lib/gridappsd/__main__.py @@ -49,19 +49,21 @@ assert sys.version_info >= (3, 6), "Minimum version is python 3.6" -logging.basicConfig(stream=sys.stdout, level=logging.INFO, +logging.basicConfig(stream=sys.stdout, + level=logging.INFO, format="'%(asctime)s: %(name)-20s - %(levelname)-6s - %(message)s") logging.getLogger('stomp.py').setLevel(logging.WARNING) _log = logging.getLogger("gridappsd.__main__") - if __name__ == '__main__': parser = ArgumentParser() group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("-s", "--run-simulation", type=argparse.FileType('r'), + group.add_argument("-s", + "--run-simulation", + type=argparse.FileType('r'), help="Start running a simulation from a passed simulation file.") opts = parser.parse_args() diff --git a/gridappsd-python-lib/gridappsd/app_registration.py b/gridappsd-python-lib/gridappsd/app_registration.py index 3ca2a6b..baf4562 100644 --- a/gridappsd-python-lib/gridappsd/app_registration.py +++ b/gridappsd-python-lib/gridappsd/app_registration.py @@ -13,8 +13,8 @@ import sys import os -from . gridappsd import GridAPPSD -from . topics import REQUEST_REGISTER_APP +from .gridappsd import GridAPPSD +from .topics import REQUEST_REGISTER_APP from . import utils, json_extension as json _log = logging.getLogger(__name__) @@ -26,6 +26,7 @@ class Job(threading.Thread): + def __init__(self, args, out=sys.stdout, err=sys.stderr): threading.Thread.__init__(self) _log.debug("Creating job") @@ -42,10 +43,7 @@ def run(self): self.running = True os.environ['GRIDAPPSD_APPLICATION_STATUS'] = 'RUNNING' - p = subprocess.Popen(args=self._args, - shell=False, - stdout=self._out, - stderr=self._err) + p = subprocess.Popen(args=self._args, shell=False, stdout=self._out, stderr=self._err) # Loop while process is executing while p.poll() is None and self.running: @@ -112,13 +110,10 @@ def heartbeat_valid(self): return self._heartbeat_thread is not None def register_app(self, end_callback): - print("Sending {}\n\tto {}".format(self._configDict, - REQUEST_REGISTER_APP)) + print("Sending {}\n\tto {}".format(self._configDict, REQUEST_REGISTER_APP)) self._gapd.get_logger().debug("Started App Registration") - response = self._gapd.get_response(REQUEST_REGISTER_APP, - self._configDict, - 60) + response = self._gapd.get_response(REQUEST_REGISTER_APP, self._configDict, 60) if 'message' in response: _log.error("An error regisering the application occured") _log.error(response.get('message')) @@ -156,7 +151,8 @@ def __start_heartbeat(self, error_callback): # print("Seanding heartbeat {} {}".format(self._heartbeat_topic, self._application_id)) # print("Heartbeat period: {}".format(self._heartbeat_period)) self._gapd.send(self._heartbeat_topic, self._application_id) - time.sleep(self._heartbeat_period - ((time.time() - starttime) % self._heartbeat_period)) + time.sleep(self._heartbeat_period - + ((time.time() - starttime) % self._heartbeat_period)) except: error_callback() diff --git a/gridappsd-python-lib/gridappsd/difference_builder.py b/gridappsd-python-lib/gridappsd/difference_builder.py index 7d48f80..a578a8f 100644 --- a/gridappsd-python-lib/gridappsd/difference_builder.py +++ b/gridappsd-python-lib/gridappsd/difference_builder.py @@ -70,7 +70,7 @@ def clear(self): self._forward = [] self._reverse = [] - def get_message(self, epoch = None): + def get_message(self, epoch=None): if epoch is None: epoch = calendar.timegm(time.gmtime()) msg = dict(command="update", diff --git a/gridappsd-python-lib/gridappsd/goss.py b/gridappsd-python-lib/gridappsd/goss.py index 8178e4a..3586804 100644 --- a/gridappsd-python-lib/gridappsd/goss.py +++ b/gridappsd-python-lib/gridappsd/goss.py @@ -37,7 +37,6 @@ # PACIFIC NORTHWEST NATIONAL LABORATORY operated by BATTELLE for the # UNITED STATES DEPARTMENT OF ENERGY under Contract DE-AC05-76RL01830 # ------------------------------------------------------------------------------- - """ Created on March 1, 2018 @@ -81,10 +80,14 @@ class GOSS(object): """ Base class providing connections to a GOSS instance via stomp protocol """ - def __init__(self, username=None, password=None, - stomp_address='localhost', stomp_port='61613', + def __init__(self, + username=None, + password=None, + stomp_address='localhost', + stomp_port='61613', attempt_connection=True, - override_threading=None, stomp_log_level=logging.WARNING, + override_threading=None, + stomp_log_level=logging.WARNING, goss_log_level=logging.INFO): logging.getLogger('stomp.py').setLevel(stomp_log_level) @@ -146,27 +149,31 @@ def send(self, topic, message): if isinstance(message, list) or isinstance(message, dict): message = json.dumps(message) _log.debug("Sending topic: {} body: {}".format(topic, message)) - self._conn.send(body=message, destination=topic, - headers={'GOSS_HAS_SUBJECT': True, - 'GOSS_SUBJECT': self.__token}) - + self._conn.send(body=message, + destination=topic, + headers={ + 'GOSS_HAS_SUBJECT': True, + 'GOSS_SUBJECT': self.__token + }) + def get_response(self, topic, message, timeout=5): id = datetime.now().strftime("%Y%m%d%h%M%S%f")[:-3] reply_to = "/temp-queue/response.{}".format(id) - + if isinstance(message, str): message = json.loads(message) - + if 'resultFormat' in message: - self.result_format = message['resultFormat'] + self.result_format = message['resultFormat'] # Change message to string if we have a dictionary. if isinstance(message, dict): message = json.dumps(message) elif isinstance(message, list): message = json.dumps(message) - + class ResponseListener(object): + def __init__(self, topic, result_format): self.response = None self._topic = topic @@ -181,7 +188,7 @@ def on_message(self, header, message): self.response = message else: self.response = json.loads(message) - else: + else: self.response = message except ValueError: self.response = dict(error="Invalid json returned", @@ -198,9 +205,13 @@ def on_disconnect(self, header, message): listener = ResponseListener(reply_to, self.result_format) self.subscribe(reply_to, listener) - self._conn.send(body=message, destination=topic, - headers={'reply-to': reply_to, 'GOSS_HAS_SUBJECT': True, - 'GOSS_SUBJECT': self.__token}) + self._conn.send(body=message, + destination=topic, + headers={ + 'reply-to': reply_to, + 'GOSS_HAS_SUBJECT': True, + 'GOSS_SUBJECT': self.__token + }) count = 0 while count < timeout: @@ -282,7 +293,7 @@ def _make_connection(self): # get token # get initial connection - dt=datetime.now() + dt = datetime.now() replyDest = f"temp.token_resp.{self.__user__}-{dt}" # create token request string @@ -297,8 +308,9 @@ def _make_connection(self): if self._override_thread_fc is not None: tmpConn.transport.override_threading(self._override_thread_fc) tmpConn.connect(self.__user__, self.__pass__, wait=True) - + class TokenResponseListener(): + def __init__(self): self.__token = None @@ -307,7 +319,7 @@ def get_token(self): def on_message(self, header, message): _log.debug("Internal on message is: {} {}".format(header, message)) - + self.__token = str(message) def on_error(self, headers, message): @@ -322,9 +334,10 @@ def on_disconnect(self, header, message): listener = TokenResponseListener() # self.subscribe(replyDest, listener) - tmpConn.subscribe('/queue/'+replyDest, 123) + tmpConn.subscribe('/queue/' + replyDest, 123) tmpConn.set_listener('token_resp', listener) - tmpConn.send(body=base64Str, destination=tokenTopic, + tmpConn.send(body=base64Str, + destination=tokenTopic, headers={'reply-to': replyDest}) # while token is null or for x iterations iter = 0 diff --git a/gridappsd-python-lib/gridappsd/gridappsd.py b/gridappsd-python-lib/gridappsd/gridappsd.py index 0664149..11b81d3 100644 --- a/gridappsd-python-lib/gridappsd/gridappsd.py +++ b/gridappsd-python-lib/gridappsd/gridappsd.py @@ -65,9 +65,9 @@ class InvalidSimulationIdError(Exception): class GridAPPSD(GOSS): """ The main :class:`GridAPPSD` interface for connecting to a GridAPPSD instance """ + # TODO Get the caller from the traceback/inspect module. - def __init__(self, simulation_id=None, - address=None, **kwargs): + def __init__(self, simulation_id=None, address=None, **kwargs): if address is None: address = utils.get_gridappsd_address() @@ -78,10 +78,7 @@ def __init__(self, simulation_id=None, 'stomp_port' in kwargs and not 'stomp_address' in kwargs: raise ValueError("If stomp_address is specified the so should stomp_port") - super(GridAPPSD, self).__init__( - stomp_address=address[0], - stomp_port=address[1], - **kwargs) + super(GridAPPSD, self).__init__(stomp_address=address[0], stomp_port=address[1], **kwargs) self._houses = Houses(self) self._simulation_log_topic = None self._simulation_id = None @@ -104,11 +101,11 @@ def get_logger(self) -> Logger: def get_houses(self) -> Houses: return self._houses - + def get_simulation_id(self): - """ + """ Return the simulation_id given to the GridAPPSD instance - + :return: """ if not self._simulation_id: @@ -116,7 +113,7 @@ def get_simulation_id(self): if self._simulation_id: self._simulation_log_topic = t.simulation_log_topic(self._simulation_id) return self._simulation_id - + def set_application_status(self, status): """ Set the application status. @@ -125,8 +122,8 @@ def set_application_status(self, status): try: self._process_status = ProcessStatusEnum(status) except ValueError: - self.get_logger().warning("Unsuccessful change of application status." - + f"Valid statuses are {ProcessStatusEnum.__members__}.") + self.get_logger().warning("Unsuccessful change of application status." + + f"Valid statuses are {ProcessStatusEnum.__members__}.") def set_service_status(self, status): """ @@ -136,8 +133,8 @@ def set_service_status(self, status): try: self._process_status = ProcessStatusEnum(status) except ValueError: - self.get_logger().warning("Unsuccessful change of service status." - + f"Valid statuses are {ProcessStatusEnum.__members__}.") + self.get_logger().warning("Unsuccessful change of service status." + + f"Valid statuses are {ProcessStatusEnum.__members__}.") def set_simulation_id(self, simulation_id): if simulation_id is None: @@ -156,17 +153,17 @@ def get_application_status(self): def get_application_id(self): return utils.get_gridappsd_application_id() - + def get_service_status(self): """ Return the service status :return: """ return self._process_status.value - + def query_object_types(self, model_id=None): """ Allows the caller to query the different object types. - + :param model_id: :return: """ @@ -186,7 +183,7 @@ def query_model_names(self, model_id=None): def query_model_info(self): payload = self._build_query_payload("QUERY_MODEL_INFO") return self.get_response(t.REQUEST_POWERGRID_DATA, payload, timeout=30) - + def query_model(self, model_id=None, object_type=None, object_id=None, response_format='JSON'): args = {} if model_id is not None: @@ -194,9 +191,9 @@ def query_model(self, model_id=None, object_type=None, object_id=None, response_ if object_type is not None: args["objectType"] = object_type if object_id is not None: - args["objectId"] = object_id + args["objectId"] = object_id if response_format is not None: - args["resultFormat"] = response_format + args["resultFormat"] = response_format payload = self._build_query_payload("QUERY_MODEL", **args) return self.get_response(t.REQUEST_POWERGRID_DATA, payload, timeout=30) @@ -208,13 +205,13 @@ def query_object(self, object_id, model_id=None): args["modelId"] = model_id payload = self._build_query_payload("QUERY_OBJECT", **args) return self.get_response(t.REQUEST_POWERGRID_DATA, payload, timeout=30) - + def query_object_dictionary(self, model_id, object_type=None, object_id=None): if not model_id: raise ValueError("model_id is not specified.") if not object_id and not object_type: raise ValueError("No obejct_id or object_type specified.") - args={} + args = {} args["modelId"] = model_id if object_id is not None: args["objectId"] = object_id @@ -235,9 +232,15 @@ def query_data(self, query, database_type=POWERGRID_MODEL, timeout=30): request_topic = '.'.join((t.REQUEST_DATA, database_type)) return self.get_response(request_topic, json.dumps(payload), timeout=timeout) - def get_platform_status(self, applications=True, services=True, appInstances=True, serviceInstances=True): + def get_platform_status(self, + applications=True, + services=True, + appInstances=True, + serviceInstances=True): _log.debug("Retrieving platform status from GridAPPSD") - msg = dict(appInstances=appInstances, applications=applications, services=services, + msg = dict(appInstances=appInstances, + applications=applications, + services=services, serviceInstances=serviceInstances) return self.get_response(t.REQUEST_PLATFORM_STATUS, json.dumps(msg), timeout=30) @@ -256,8 +259,7 @@ def send_simulation_status(self, status, message, log_level=INFO): def send_status(self, status, topic, log_level=INFO): self.set_application_status(status) - status_message = self.build_message_json(self.get_application_status(), - "", log_level) + status_message = self.build_message_json(self.get_application_status(), "", log_level) self.send(topic, status_message) def build_message_json(self, status, message, log_level): @@ -265,7 +267,7 @@ def build_message_json(self, status, message, log_level): status_message = { "source": utils.get_gridappsd_application_id(), "processId": f"{self._simulation_id}", - "timestamp": int(time.mktime(t_now.timetuple()))*1000, + "timestamp": int(time.mktime(t_now.timetuple())) * 1000, "procesStatus": status, "logMessage": str(message), "logLevel": logging.getLevelName(log_level), @@ -279,6 +281,3 @@ def _build_query_payload(self, request_type, response_format='JSON', **kwargs): d = dict(requestType=request_type, resultFormat=response_format) d.update(**kwargs) return d - - - diff --git a/gridappsd-python-lib/gridappsd/houses.py b/gridappsd-python-lib/gridappsd/houses.py index 39b7d7c..076eef9 100644 --- a/gridappsd-python-lib/gridappsd/houses.py +++ b/gridappsd-python-lib/gridappsd/houses.py @@ -1,18 +1,8 @@ from collections import namedtuple house_keys = [ - 'name', - 'parent', - 'coolingSetpoint', - 'coolingSystem', - 'floorArea', - 'heatingSetpoint', - 'heatingSystem', - 'hvacPowerFactor', - 'numberOfStories', - 'thermalIntegrity', - 'id', - 'fdrid' + 'name', 'parent', 'coolingSetpoint', 'coolingSystem', 'floorArea', 'heatingSetpoint', + 'heatingSystem', 'hvacPowerFactor', 'numberOfStories', 'thermalIntegrity', 'id', 'fdrid' ] House = namedtuple('House', house_keys) @@ -21,7 +11,9 @@ class Houses: + class __SingltonHouses: + def __init__(self, gappsd: 'GridAPPSD'): self._gappsd = gappsd self._houses = {} @@ -38,8 +30,8 @@ def _populate(self, feeder): query = """# list houses - DistHouse PREFIX r: PREFIX c: -SELECT ?fdrname ?name ?parent ?coolingSetpoint ?coolingSystem ?floorArea ?heatingSetpoint ?heatingSystem ?hvacPowerFactor ?numberOfStories ?thermalIntegrity ?id ?fdrid -WHERE { +SELECT ?fdrname ?name ?parent ?coolingSetpoint ?coolingSystem ?floorArea ?heatingSetpoint ?heatingSystem ?hvacPowerFactor ?numberOfStories ?thermalIntegrity ?id ?fdrid +WHERE { # VALUES ?fdrid {"_9CE150A8-8CC5-A0F9-B67E-BBD8C79D3095"} # R2 12.47 3 # VALUES ?fdrid {"_4F76A5F9-271D-9EB8-5E31-AA362D86F2C3"} # 8500-node # VALUES ?fdrid {"_E407CBB6-8C8D-9BC9-589C-AB83FBF0826D"} # 123 PV/Triplex @@ -54,7 +46,7 @@ def _populate(self, feeder): OPTIONAL{?h c:House.heatingSetpoint ?heatingSetpoint.} OPTIONAL{?h c:House.hvacPowerFactor ?hvacPowerFactor.} ?h c:House.coolingSystem ?coolingSystemRaw. - bind(strafter(str(?coolingSystemRaw),"HouseCooling.") as ?coolingSystem) + bind(strafter(str(?coolingSystemRaw),"HouseCooling.") as ?coolingSystem) ?h c:House.heatingSystem ?heatingSystemRaw. bind(strafter(str(?heatingSystemRaw),"HouseHeating.") as ?heatingSystem) ?h c:House.thermalIntegrity ?thermalIntegrityRaw. @@ -83,7 +75,6 @@ def _populate(self, feeder): self._houses[feeder] = houses - instance = None def __init__(self, gappsd): @@ -94,4 +85,3 @@ def __init__(self, gappsd): def __getattr__(self, name): return getattr(self.instance, name) - diff --git a/gridappsd-python-lib/gridappsd/json_extension.py b/gridappsd-python-lib/gridappsd/json_extension.py index 0c92bb3..81d7343 100644 --- a/gridappsd-python-lib/gridappsd/json_extension.py +++ b/gridappsd-python-lib/gridappsd/json_extension.py @@ -21,13 +21,11 @@ def jsonDecoderExtension(obj: Any): class JsonEncoderExtension(_json.JSONEncoder): + def default(self, obj: Any) -> Any: rv = None if isinstance(obj, complex): - jsonComplexInstance = JsonComplex( - real = obj.real, - imag = obj.imag - ) + jsonComplexInstance = JsonComplex(real=obj.real, imag=obj.imag) rv = dataclasses.asdict(jsonComplexInstance) elif dataclasses.is_dataclass(obj): rv = dataclasses.asdict(obj) @@ -52,4 +50,4 @@ def load(fo: TextIO) -> Any: def loads(data: str) -> Any: rv = _json.loads(data, object_hook=jsonDecoderExtension) - return rv \ No newline at end of file + return rv diff --git a/gridappsd-python-lib/gridappsd/loghandler.py b/gridappsd-python-lib/gridappsd/loghandler.py index b79f135..a2997ef 100644 --- a/gridappsd-python-lib/gridappsd/loghandler.py +++ b/gridappsd-python-lib/gridappsd/loghandler.py @@ -35,6 +35,7 @@ class Logger: """ The `Logger` class handles logging to the main gridappsd server. """ + def __init__(self, gaps, level=INFO): self._gaps = gaps self._level = level @@ -71,8 +72,9 @@ def log(self, message, level=DEBUG): raise AttributeError(f"Log level must be one of {[x for x in _levelToName.values()]}") if not process_identifier: - raise AttributeError(f"Must have GRIDAPPSD_APPLICATION_ID or GRIDAPPSD_SERVICE_ID or GRIDAPPSD_PROCESS_ID " - "set in os environments.") + raise AttributeError( + f"Must have GRIDAPPSD_APPLICATION_ID or GRIDAPPSD_SERVICE_ID or GRIDAPPSD_PROCESS_ID " + "set in os environments.") status = self._gaps.get_application_status() sim_id = self._gaps.get_simulation_id() @@ -89,13 +91,12 @@ def log(self, message, level=DEBUG): "logLevel": _levelToName[level], "storeToDb": True } - + gridappsd_log_level = os.getenv('GRIDAPPSD_LOG_LEVEL') if gridappsd_log_level == None: gridappsd_log_level = level else: gridappsd_log_level = _nameToLevel[gridappsd_log_level] - + if level >= gridappsd_log_level: self._gaps.send(topic, status_message) - diff --git a/gridappsd-python-lib/gridappsd/register_app.py b/gridappsd-python-lib/gridappsd/register_app.py index 7ca3da0..1411816 100644 --- a/gridappsd-python-lib/gridappsd/register_app.py +++ b/gridappsd-python-lib/gridappsd/register_app.py @@ -5,9 +5,11 @@ import stomp from gridappsd import ApplicationController, GridAPPSD, utils, json_extension as json + def main(): loglevel = logging.INFO - logging.basicConfig(stream=sys.stdout, level=loglevel, + logging.basicConfig(stream=sys.stdout, + level=loglevel, format="%(asctime)s - %(name)s;%(levelname)s|%(message)s", datefmt="%Y-%m-%d %H:%M:%S") logging.getLogger('stomp.py').setLevel(logging.ERROR) @@ -43,7 +45,7 @@ def main(): if gap is None: gap = GridAPPSD() - except ConnectionRefusedError: # Python 3 specific error code + except ConnectionRefusedError: # Python 3 specific error code _log.debug("Retry in 10 seconds") gap = appreg = None time.sleep(10) @@ -57,6 +59,7 @@ def main(): break else: if appreg is None: + def end_app(): sys.exit(0) @@ -78,5 +81,3 @@ def end_app(): gap = None time.sleep(2) - - diff --git a/gridappsd-python-lib/gridappsd/simulation.py b/gridappsd-python-lib/gridappsd/simulation.py index 2edfa3c..8ff94b0 100644 --- a/gridappsd-python-lib/gridappsd/simulation.py +++ b/gridappsd-python-lib/gridappsd/simulation.py @@ -120,10 +120,8 @@ class Simulation: add_onmeasurement_callback, add_oncomplete_callback or add_onstart_callback method respectively. """ - def __init__(self, gapps: 'GridAPPSD', - run_config: Union[Dict, SimulationConfig]): - assert type( - gapps).__name__ == 'GridAPPSD', "Must be an instance of GridAPPSD" + def __init__(self, gapps: 'GridAPPSD', run_config: Union[Dict, SimulationConfig]): + assert type(gapps).__name__ == 'GridAPPSD', "Must be an instance of GridAPPSD" self._run_config = run_config # if isinstance(run_config, SimulationConfig): @@ -157,8 +155,7 @@ def __init__(self, gapps: 'GridAPPSD', self._log_count = 0 self._platform_log_count = 0 - self._num_timesteps = round( - float(self._run_config["simulation_config"]["duration"])) + self._num_timesteps = round(float(self._run_config["simulation_config"]["duration"])) # self._num_timesteps = round( # float(self._run_config.simulation_config.duration)) @@ -171,9 +168,7 @@ def __init__(self, gapps: 'GridAPPSD', def start_simulation(self, timeout=30): """ Start the configured simulation by calling the REQUEST_SIMULATION endpoint. """ - resp = self._gapps.get_response(t.REQUEST_SIMULATION, - self._run_config, - timeout=timeout) + resp = self._gapps.get_response(t.REQUEST_SIMULATION, self._run_config, timeout=timeout) if 'simulationId' not in resp: message = "Simulation was not able to run\n" + str(resp) @@ -184,10 +179,8 @@ def start_simulation(self, timeout=30): # Subscribe to the different components necessary to run and receive # simulated measurements and messages. - self._gapps.subscribe(t.simulation_log_topic(self.simulation_id), - self.__on_simulation_log) - self._gapps.subscribe(t.simulation_output_topic(self.simulation_id), - self.__onmeasurement) + self._gapps.subscribe(t.simulation_log_topic(self.simulation_id), self.__on_simulation_log) + self._gapps.subscribe(t.simulation_output_topic(self.simulation_id), self.__onmeasurement) self._gapps.subscribe(t.platform_log_topic(), self.__on_platformlog) for p in self.__on_start: @@ -197,24 +190,21 @@ def pause(self): """ Pause simulation""" _log.debug("Pausing simulation") command = dict(command="pause") - self._gapps.send(t.simulation_input_topic(self.simulation_id), - json.dumps(command)) + self._gapps.send(t.simulation_input_topic(self.simulation_id), json.dumps(command)) self._running_or_paused = True def stop(self): """ Stop the simulation""" _log.debug("Stopping simulation") command = dict(command="stop") - self._gapps.send(t.simulation_input_topic(self.simulation_id), - json.dumps(command)) + self._gapps.send(t.simulation_input_topic(self.simulation_id), json.dumps(command)) self._running_or_paused = True def resume(self): """ Resume the simulation""" _log.debug("Resuming simulation") command = dict(command="resume") - self._gapps.send(t.simulation_input_topic(self.simulation_id), - json.dumps(command)) + self._gapps.send(t.simulation_input_topic(self.simulation_id), json.dumps(command)) self._running_or_paused = True def run_loop(self): @@ -242,14 +232,12 @@ def run_loop(self): def resume_pause_at(self, pause_in): """ Resume the simulation and have it automatically pause after specified amount of seconds later. - + :param pause_in: number of seconds to run before pausing the simulation """ - _log.debug("Resuming simulation. Will pause after {} seconds".format( - pause_in)) + _log.debug("Resuming simulation. Will pause after {} seconds".format(pause_in)) command = dict(command="resumePauseAt", input=dict(pauseIn=pause_in)) - self._gapps.send(t.simulation_input_topic(self.simulation_id), - json.dumps(command)) + self._gapps.send(t.simulation_input_topic(self.simulation_id), json.dumps(command)) self._running_or_paused = True def add_onmeasurement_callback(self, callback, device_filter=()): @@ -271,8 +259,7 @@ def onmeasurment(sim, timestep, measurements): :param device_filter: Future filter of measurements :return: """ - self.__filterable_measurement_callback_set.add( - (callback, device_filter)) + self.__filterable_measurement_callback_set.add((callback, device_filter)) def add_onstart_callback(self, callback): """ registers a start callback that is called when the simulation is started @@ -361,4 +348,4 @@ def __onmeasurement(self, headers, message): print(psc.asjson()) print(sim.asjson()) - pprint(json.loads(sim.asjson()), indent=2) \ No newline at end of file + pprint(json.loads(sim.asjson()), indent=2) diff --git a/gridappsd-python-lib/gridappsd/timeseries.py b/gridappsd-python-lib/gridappsd/timeseries.py index a8128b8..d2ce8b1 100644 --- a/gridappsd-python-lib/gridappsd/timeseries.py +++ b/gridappsd-python-lib/gridappsd/timeseries.py @@ -3,11 +3,10 @@ from typing import Optional - class Query: """ Class to create and execute request to query timeseries data """ - + def __init__(self, measurement): self.queryMeasurement = measurement self.queryFilter = [] @@ -16,11 +15,11 @@ def __init__(self, measurement): def select(self, *keys): """ Defines what fields should be returned from the query. - - If this function is not called or is called without argument then + + If this function is not called or is called without argument then all the fields are returned. - - :param keys: list of fields to be returned + + :param keys: list of fields to be returned """ for key in keys: self.selectCriteria.append(key) @@ -28,78 +27,79 @@ def select(self, *keys): def first(self, n=Optional[int]): """ Method to add request to return first or oldest data to the query. - + When n is specified, query will return first or oldest 'n' rows. - - :param n: + + :param n: """ if n is not None: - self.first = n; + self.first = n return self def last(self, n=Optional[int]): """ Method to add request to return last or latest data to the query. - + When n is specified, query will return last or latest 'n' rows. - - :param n: + + :param n: """ if n is not None: - self.last = n; + self.last = n return self - + def ge(self, value): """ Method to add 'value greater than or equal to' filter to a key. - - :param value: + + :param value: """ if self.key is None: raise ValueError("Key is not specified. Call where_key first.") - obj = {"key":self.key,"ge":value} + obj = {"key": self.key, "ge": value} self.queryFilter.append(obj) return self - + def le(self, value): """ Method to add 'value less than or equal to' filter to a key. - - :param value: + + :param value: """ if self.key is None: raise ValueError("Key is not specified. Call where_key first.") - obj = {"key":self.key,"le":value} + obj = {"key": self.key, "le": value} self.queryFilter.append(obj) return self - + def eq(self, value): """ Method to add 'value equal to' filter to a key. - - :param value: + + :param value: """ if self.key is None: raise ValueError("Key is not specified. Call where_key first.") - obj = {"key":self.key,"eq":value} + obj = {"key": self.key, "eq": value} self.queryFilter.append(obj) return self - + def between(self, value1, value2): """ Method to add 'value between' value1 and value2 filter to a key. - + :param value1: defines 'greater than equal to' filter for key's value :param value2: defines 'less than equal to' filter for key's value """ if self.key is None: raise ValueError("Key is not specified. Call where_key first.") - obj = {"key":self.key,"ge":value1} + obj = {"key": self.key, "ge": value1} self.queryFilter.append(obj) - obj = {"key":self.key,"le":value2} + obj = {"key": self.key, "le": value2} self.queryFilter.append(obj) return self - + def where_key(self, key): self.key = key return self def execute(self, gridappsd_obj): - del self.key - response = gridappsd_obj.get_response(topics.REQUEST_TIMESERIES_DATA, json.dumps(self.__dict__)) - return response \ No newline at end of file + del self.key + response = gridappsd_obj.get_response(topics.REQUEST_TIMESERIES_DATA, + json.dumps(self.__dict__)) + return response diff --git a/gridappsd-python-lib/gridappsd/topics.py b/gridappsd-python-lib/gridappsd/topics.py index 2934abe..aeead75 100644 --- a/gridappsd-python-lib/gridappsd/topics.py +++ b/gridappsd-python-lib/gridappsd/topics.py @@ -73,6 +73,7 @@ REQUEST_APP_START = ".".join((PROCESS_PREFIX, "request.app.start")) BASE_APPLICATION_HEARTBEAT = ".".join((BASE_TOPIC_PREFIX, "heartbeat")) + def platform_log_topic(): """ Utility method for getting the platform.log base topic """ @@ -175,7 +176,8 @@ def simulation_log_topic(simulation_id): """ return "{}.{}".format(BASE_SIMULATION_LOG_TOPIC, simulation_id) -def field_message_bus_topic(message_bus_id:str, app_id: str=None, agent_id: str=None): + +def field_message_bus_topic(message_bus_id: str, app_id: str = None, agent_id: str = None): """ Utility method for getting the publish/subscribe topic for a specific message bus. :param message_bus_id: @@ -185,7 +187,7 @@ def field_message_bus_topic(message_bus_id:str, app_id: str=None, agent_id: str= """ assert message_bus_id, "message_bus_id cannot be empty" - return f"{BASE_FIELD_TOPIC}.{message_bus_id}" + return f"{BASE_FIELD_TOPIC}.{message_bus_id}" def field_message_bus_app_topic(message_bus_id, app_id=None): @@ -198,6 +200,7 @@ def field_message_bus_app_topic(message_bus_id, app_id=None): assert message_bus_id, "message_bus_id cannot be empty" return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, app_id) + def field_message_bus_agent_topic(message_bus_id, agent_id=None): """ Utility method for getting the publish/subscribe topic for a specific message bus. @@ -208,6 +211,7 @@ def field_message_bus_agent_topic(message_bus_id, agent_id=None): assert message_bus_id, "message_bus_id cannot be empty" return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, agent_id) + def field_agent_request_queue(message_bus_id, agent_id): """ Utility method for getting the request topic for a specific distributed agent @@ -218,6 +222,7 @@ def field_agent_request_queue(message_bus_id, agent_id): assert message_bus_id, "message_bus_id cannot be empty" return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, agent_id) + def context_request_queue(message_bus_id): """ Utility method for getting the request topic for context manager @@ -225,35 +230,40 @@ def context_request_queue(message_bus_id): :return: """ assert message_bus_id, "message_bus_id cannot be empty" - - return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, message_bus_id+'.context_manager') + + return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, + message_bus_id + '.context_manager') + def field_output_topic(message_bus_id=None, simulation_id=None): """ Utility method for getting the field output topic. If message_bus_id is None, it returns topic used by centralized device interfaces to publish measurements. - If message_bus_id is not None, it returns topic used by distributed devices interfaces to publish measurements which is then subscribed by distributed agents. - + If message_bus_id is not None, it returns topic used by distributed devices interfaces to publish measurements which is then subscribed by distributed agents. + :param message_bus_id: :param simulation_id - :return: str: Topic to receive field measurements + :return: str: Topic to receive field measurements """ - + if simulation_id is None: return "{}.{}".format(BASE_FIELD_TOPIC, "output") else: - return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.output",simulation_id,message_bus_id) - + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC, "simulation.output", simulation_id, + message_bus_id) + + def field_input_topic(message_bus_id=None, simulation_id=None): """ Utility method for getting the field input topic. If message_bus_id is None, it returns topic used by centralized device interfaces to subscribe to control commands. - If message_bus_id is not None, it returns topic used by distributed devices interfaces to subscribe to control commands. - + If message_bus_id is not None, it returns topic used by distributed devices interfaces to subscribe to control commands. + :param message_bus_id: :param simulation_id - :return: str: Topic to receive input control commands + :return: str: Topic to receive input control commands """ - + if simulation_id is None: return "{}.{}".format(BASE_FIELD_TOPIC, "input") else: - return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.input",simulation_id,message_bus_id) + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC, "simulation.input", simulation_id, + message_bus_id) diff --git a/gridappsd-python-lib/gridappsd/utils.py b/gridappsd-python-lib/gridappsd/utils.py index 98c917f..ab718bf 100644 --- a/gridappsd-python-lib/gridappsd/utils.py +++ b/gridappsd-python-lib/gridappsd/utils.py @@ -4,12 +4,11 @@ from dateutil import parser import os -try: # python2.7 +try: # python2.7 from urlparse import urlparse except ImportError: from urllib.parse import urlparse - __GRIDAPPSD_URI__ = os.environ.get("GRIDAPPSD_URI", "localhost:61613") if not __GRIDAPPSD_URI__.startswith("tcp://"): @@ -68,8 +67,7 @@ def get_gridappsd_address(): return address, port - return (__GRIDAPPSD_URI_PARSED__.hostname, - __GRIDAPPSD_URI_PARSED__.port) + return (__GRIDAPPSD_URI_PARSED__.hostname, __GRIDAPPSD_URI_PARSED__.port) def get_gridappsd_application_id(): diff --git a/gridappsd-python-lib/info/VERSION b/gridappsd-python-lib/info/VERSION index 17c91dc..dd41647 100644 --- a/gridappsd-python-lib/info/VERSION +++ b/gridappsd-python-lib/info/VERSION @@ -1 +1 @@ -2023.5.1 \ No newline at end of file +2023.5.1 diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index 23a861a..33111b0 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -56,7 +56,7 @@ ignore_patterns = [ "dist/**", "docs/**" ] - + [tool.yapf] based_on_style = "pep8" spaces_before_comment = 4 diff --git a/gridappsd-python-lib/tests/conftest.py b/gridappsd-python-lib/tests/conftest.py index 5319cc8..2e984dd 100644 --- a/gridappsd-python-lib/tests/conftest.py +++ b/gridappsd-python-lib/tests/conftest.py @@ -15,14 +15,7 @@ # from gridappsd.docker_handler import (Containers, run_dependency_containers, # run_gridappsd_container) -levels = dict(CRITICAL=50, - FATAL=50, - ERROR=40, - WARNING=30, - WARN=30, - INFO=20, - DEBUG=10, - NOTSET=0) +levels = dict(CRITICAL=50, FATAL=50, ERROR=40, WARNING=30, WARN=30, INFO=20, DEBUG=10, NOTSET=0) # Get string representation of the log level passed LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG") @@ -43,8 +36,7 @@ _log = logging.getLogger(__name__) -STOP_CONTAINER_AFTER_TEST = os.environ.get( - 'GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) +STOP_CONTAINER_AFTER_TEST = os.environ.get('GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) os.environ['GRIDAPPSD_USER'] = 'system' os.environ['GRIDAPPSD_PASSWORD'] = 'manager' os.environ['GRIDAPPSD_TAG'] = ':develop' @@ -115,7 +107,6 @@ def docker_compose_up() -> DockerClient: def gridappsd_client(request, docker_compose_up: DockerClient): dc = docker_compose_up - gappsd = GridAPPSD() assert gappsd.connected diff --git a/gridappsd-python-lib/tests/run_gridappsd.py b/gridappsd-python-lib/tests/run_gridappsd.py index fd113e2..7e7a391 100644 --- a/gridappsd-python-lib/tests/run_gridappsd.py +++ b/gridappsd-python-lib/tests/run_gridappsd.py @@ -21,8 +21,8 @@ while gapps is None: try: gapps = GridAPPSD() - except (ConnectionRefusedError, stomp.exception.ConnectFailedException, - socket.gaierror, OSError): + except (ConnectionRefusedError, stomp.exception.ConnectFailedException, socket.gaierror, + OSError): _log.debug("Not Connected") time.sleep(5) else: diff --git a/gridappsd-python-lib/tests/test_docker_handler.py b/gridappsd-python-lib/tests/test_docker_handler.py index 4f8826b..6f2d2b7 100644 --- a/gridappsd-python-lib/tests/test_docker_handler.py +++ b/gridappsd-python-lib/tests/test_docker_handler.py @@ -17,7 +17,6 @@ # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) # _log = logging.getLogger(inspect.getmodulename(__file__)) - # def test_log_container(docker_dependencies): # mypath = "/tmp/alphabetagamma.txt" # stream_container_log_to_file("influxdb", mypath) @@ -27,7 +26,6 @@ # with open(mypath, 'rb') as rf: # assert len(rf.readlines()) > 0 - # def test_can_reset_all_containers(): # Containers.reset_all_containers() # assert not Containers.container_list() @@ -53,7 +51,6 @@ # Containers.reset_all_containers() # assert not Containers.container_list() - # def test_can_dependencies_continue_after_context_manager(): # my_config = DEFAULT_DOCKER_DEPENDENCY_CONFIG.copy() # Containers.reset_all_containers() @@ -75,7 +72,6 @@ # Containers.reset_all_containers() - # def test_create_volume_container(): # Containers.create_volume_container("test_volume", # "test_volume", @@ -87,7 +83,6 @@ # result = client.containers.get("test_volume").exec_run("ls -l /startup") # assert True - # def test_can_upload_files_to_container(): # Containers.reset_all_containers() @@ -106,7 +101,6 @@ # for f in os.listdir(conf_path): # assert f in results.output.decode("utf-8"), f"{f} was not in /conf" - # def test_multiple_runs_in_a_row_with_dependency_context_manager(): # Containers.reset_all_containers() @@ -151,7 +145,6 @@ # assert gapps # assert gapps.connected - # def test_can_start_gridappsd_within_dependency_context_manager_all_cleanup(): # Containers.reset_all_containers() @@ -181,7 +174,6 @@ # ] # assert not len(containers) - # def test_can_start_gridapps(): # Containers.reset_all_containers() # with run_dependency_containers() as cont: @@ -189,7 +181,6 @@ # g = GridAPPSD() # assert g.connected - # def test_mysql_setup(): # mysql_setup() # assert Path(MYSQL_SCHEMA_INIT_DIR).exists() diff --git a/gridappsd-python-lib/tests/test_goss.py b/gridappsd-python-lib/tests/test_goss.py index 7d5c4f4..caab642 100644 --- a/gridappsd-python-lib/tests/test_goss.py +++ b/gridappsd-python-lib/tests/test_goss.py @@ -14,7 +14,6 @@ # _log = logging.getLogger(__name__) - # def test_auth_raises_error_no_username_password(docker_dependencies): # container = get_docker_in_docker() # mockdict = { @@ -34,7 +33,6 @@ # with pytest.raises(ValueError) as ex: # goss = GOSS(password="bar") - # def test_get_response(caplog, goss_client): # caplog.set_level(logging.DEBUG) @@ -85,7 +83,6 @@ # assert len(gen_sub[0]) == 2 # assert result['result'] == 11 - # def test_send_receive(goss_client): # message_queue = Queue() @@ -101,7 +98,6 @@ # header, message = message_queue.get() # assert message == "I am a foo" - # def test_callback_function(goss_client): # message_queue1 = Queue() @@ -115,7 +111,6 @@ # header, message = message_queue1.get() # assert message == "I am a foo" - # def test_multi_subscriptions(goss_client): # message_queue1 = Queue() # message_queue2 = Queue() @@ -141,7 +136,6 @@ # header, message = message_queue2.get() # assert message == "I am a bar" - # def test_multi_subscriptions_same_topic(goss_client): # # pytest.xfail("Multiple topics can't be subscribed to the same topic at present.") @@ -173,7 +167,6 @@ # header, message = message_queue2.get() # assert message == "I am a bar" - # def test_response_class(goss_client): # message_queue = Queue() @@ -193,7 +186,6 @@ # assert dict(abc="def") == result[1] - # def test_replace_subscription(caplog, goss_client): # caplog.set_level(logging.DEBUG) # original_queue = Queue() diff --git a/gridappsd-python-lib/tests/test_gridappsd.py b/gridappsd-python-lib/tests/test_gridappsd.py index 6990cf8..ae82046 100644 --- a/gridappsd-python-lib/tests/test_gridappsd.py +++ b/gridappsd-python-lib/tests/test_gridappsd.py @@ -9,133 +9,131 @@ #, topics as t, ProcessStatusEnum + # def test_get_gridappsd_client(gridappsd_client: GridAPPSD): + # assert isinstance(gridappsd_client, GridAPPSD) -# def test_get_gridappsd_client(gridappsd_client: GridAPPSD): -# assert isinstance(gridappsd_client, GridAPPSD) + # def test_get_model_info(gridappsd_client): + # """ The expecation is that we will have multiple models that we can retrieve from the + # database. Two of which should have the model name of ieee8500 and ieee123. The models + # should have the correct entry keys. + # """ + # gappsd = gridappsd_client + # import time + # time.sleep(10) + # info = gappsd.query_model_info() + + # node_8500 = None + # node_123 = None + # for info_def in info['data']['models']: + # if info_def['modelName'] == 'ieee8500': + # node_8500 = info_def + # elif info_def['modelName'] == 'ieee123': + # node_123 = info_def + + # assert node_123, "Missing the 123 model" + # assert node_8500, "Missing 8500 node model." + + # keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", + # "regionName", "regionId"] + # correct_keys = set(keys) + + # assert len(correct_keys) == len(node_123) + # assert len(correct_keys) == len(node_8500) + + # for x in node_123: + # correct_keys.remove(x) + + # assert len(correct_keys) == 0 + + # correct_keys = set(keys) + + # for x in node_8500: + # correct_keys.remove(x) + + # assert len(correct_keys) == 0 + + # def test_listener_multi_topic(gridappsd_client): + # gappsd = gridappsd_client + + # class Listener: + # def __init__(self): + # self.call_count = 0 + + # def reset(self): + # self.call_count = 0 + + # def on_message(self, headers, message): + # print("Message was: {}".format(message)) + # self.call_count += 1 + + # listener = Listener() + + # input_topic = t.simulation_input_topic("5144") + # output_topic = t.simulation_output_topic("5144") + + # gappsd.subscribe(input_topic, listener) + # gappsd.subscribe(output_topic, listener) + + # gappsd.send(input_topic, "Any message") + # sleep(1) + # assert 1 == listener.call_count + # listener.reset() + # gappsd.send(output_topic, "No big deal") + # sleep(1) + # assert 1 == listener.call_count + + # @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", + # "GRIDAPPSD_SIMULATION_ID": "1234"}) + # def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): + + # class Listener: + # def __init__(self): + # self.call_count = 0 + + # def reset(self): + # self.call_count = 0 + + # def on_message(self, headers, message): + # print("Message was: {}".format(message)) + # self.call_count += 1 + + # listener = Listener() + # gappsd = gridappsd_client + # assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' + # assert gappsd.get_simulation_id() == "1234" + + # log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) + # gappsd.subscribe(log_topic, listener) + # gappsd.send_simulation_status("RUNNING", + # "testing the sending and recieving of send_simulation_status().", + # logging.DEBUG) + # sleep(1) + # assert listener.call_count == 1 + + # new_log_topic = t.simulation_log_topic("54232") + # gappsd.set_simulation_id(54232) + # gappsd.subscribe(new_log_topic, listener) + # gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") + # sleep(1) + # assert listener.call_count == 2 + + # @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) + # def test_gridappsd_status(gridappsd_client): + # gappsd = gridappsd_client + # assert "helics_goss_bridge.py" == gappsd.get_application_id() + # assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value + # assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value + # gappsd.set_application_status("RUNNING") + + # assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value + # assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value -# def test_get_model_info(gridappsd_client): -# """ The expecation is that we will have multiple models that we can retrieve from the -# database. Two of which should have the model name of ieee8500 and ieee123. The models -# should have the correct entry keys. -# """ + # gappsd.set_service_status("COMPLETE") + # assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + # assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value -# gappsd = gridappsd_client -# import time -# time.sleep(10) -# info = gappsd.query_model_info() - -# node_8500 = None -# node_123 = None -# for info_def in info['data']['models']: -# if info_def['modelName'] == 'ieee8500': -# node_8500 = info_def -# elif info_def['modelName'] == 'ieee123': -# node_123 = info_def - -# assert node_123, "Missing the 123 model" -# assert node_8500, "Missing 8500 node model." - -# keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", -# "regionName", "regionId"] -# correct_keys = set(keys) - -# assert len(correct_keys) == len(node_123) -# assert len(correct_keys) == len(node_8500) - -# for x in node_123: -# correct_keys.remove(x) - -# assert len(correct_keys) == 0 - -# correct_keys = set(keys) - -# for x in node_8500: -# correct_keys.remove(x) - -# assert len(correct_keys) == 0 - -# def test_listener_multi_topic(gridappsd_client): -# gappsd = gridappsd_client - -# class Listener: -# def __init__(self): -# self.call_count = 0 - -# def reset(self): -# self.call_count = 0 - -# def on_message(self, headers, message): -# print("Message was: {}".format(message)) -# self.call_count += 1 - -# listener = Listener() - -# input_topic = t.simulation_input_topic("5144") -# output_topic = t.simulation_output_topic("5144") - -# gappsd.subscribe(input_topic, listener) -# gappsd.subscribe(output_topic, listener) - -# gappsd.send(input_topic, "Any message") -# sleep(1) -# assert 1 == listener.call_count -# listener.reset() -# gappsd.send(output_topic, "No big deal") -# sleep(1) -# assert 1 == listener.call_count - -# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", -# "GRIDAPPSD_SIMULATION_ID": "1234"}) -# def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): - -# class Listener: -# def __init__(self): -# self.call_count = 0 - -# def reset(self): -# self.call_count = 0 - -# def on_message(self, headers, message): -# print("Message was: {}".format(message)) -# self.call_count += 1 - -# listener = Listener() -# gappsd = gridappsd_client -# assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' -# assert gappsd.get_simulation_id() == "1234" - -# log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) -# gappsd.subscribe(log_topic, listener) -# gappsd.send_simulation_status("RUNNING", -# "testing the sending and recieving of send_simulation_status().", -# logging.DEBUG) -# sleep(1) -# assert listener.call_count == 1 - -# new_log_topic = t.simulation_log_topic("54232") -# gappsd.set_simulation_id(54232) -# gappsd.subscribe(new_log_topic, listener) -# gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") -# sleep(1) -# assert listener.call_count == 2 - -# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) -# def test_gridappsd_status(gridappsd_client): -# gappsd = gridappsd_client -# assert "helics_goss_bridge.py" == gappsd.get_application_id() -# assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value -# assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value -# gappsd.set_application_status("RUNNING") - -# assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value -# assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value - -# gappsd.set_service_status("COMPLETE") -# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value -# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - -# # Invalid -# gappsd.set_service_status("Foo") -# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value -# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value + # # Invalid + # gappsd.set_service_status("Foo") + # assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + # assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value diff --git a/gridappsd-python-lib/tests/test_houses.py b/gridappsd-python-lib/tests/test_houses.py index c9eb6ba..6523fe2 100644 --- a/gridappsd-python-lib/tests/test_houses.py +++ b/gridappsd-python-lib/tests/test_houses.py @@ -1,4 +1,3 @@ - # # def test_multiple_calls_gets_same_houses_obj(gridappsd_client): # gapps = gridappsd_client @@ -7,7 +6,6 @@ # houses_again = gapps.get_houses() # assert id_of_houses == id(houses_again) - # def test_can_get_transactive_houses(gridappsd_client): # # houses = gridappsd_client.get_houses().get_houses_for_feeder('_503D6E20-F499-4CC7-8051-971E23D0BF79') diff --git a/gridappsd-python-lib/tests/test_json.py b/gridappsd-python-lib/tests/test_json.py index 89de077..d00f9b7 100644 --- a/gridappsd-python-lib/tests/test_json.py +++ b/gridappsd-python-lib/tests/test_json.py @@ -1,11 +1,12 @@ from gridappsd import json + def test_json_complex(): message = { "key1": { - "key2": complex(3.369,4.213), + "key2": complex(3.369, 4.213), "key3": { - "key4": complex(-5.147,-6.258), + "key4": complex(-5.147, -6.258), "key5": { "real": -9.654, "imag": 8.321 @@ -22,9 +23,9 @@ def test_json_complex(): serializedMessage = """{"key1": {"key2": {"real": 3.369, "imag": 4.213}, "key3": {"key4": {"real": -5.147, "imag": -6.258}, "key5": {"real": -9.654, "imag": 8.321}}, "key6": {"real": 7.894, "imag": -8.542, "garbage": true}}, "key7": {"real": 7.894, "imag": -8.542}}""" deserializedMessage = { "key1": { - "key2": complex(3.369,4.213), + "key2": complex(3.369, 4.213), "key3": { - "key4": complex(-5.147,-6.258), + "key4": complex(-5.147, -6.258), "key5": complex(-9.654, 8.321), }, "key6": { @@ -38,4 +39,4 @@ def test_json_complex(): encodedStr = json.dumps(message) assert encodedStr == serializedMessage decodedMessage = json.loads(encodedStr) - assert decodedMessage == deserializedMessage \ No newline at end of file + assert decodedMessage == deserializedMessage diff --git a/gridappsd-python-lib/tests/test_logging.py b/gridappsd-python-lib/tests/test_logging.py index a4da7ee..883db0c 100644 --- a/gridappsd-python-lib/tests/test_logging.py +++ b/gridappsd-python-lib/tests/test_logging.py @@ -5,7 +5,6 @@ # from mock import Mock # import pytest - # def init_gapps_mock(simulation_id=None, application_id=None, process_status=None, service_id=None): # gapps = Mock() @@ -16,7 +15,6 @@ # return gapps - # #@mock.patch('gridappsd.utils.get_application_id') # def test_required_application_id_set(): # """ os.environ['GRIDAPPSD_APPLICATION_ID'] must be set to run.""" @@ -25,7 +23,6 @@ # with pytest.raises(AttributeError): # log.debug("foo") - # def test_no_simulation_id_topic_or_application_id(): # """If no simulation then the topic should be the platform log topic""" # expected_topic = t.platform_log_topic() @@ -42,7 +39,6 @@ # assert message['processStatus'] == ProcessStatusEnum.STARTING.value # assert message['logMessage'] == 'A message' - # def test_platform_log(): # application_id = "my_app" @@ -87,7 +83,6 @@ # assert 'WARN' == message['logLevel'] # assert 'baf' == message['logMessage'] - # def test_invalid_log_level(): # application_id = "my_app" # gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) @@ -96,7 +91,6 @@ # with pytest.raises(AttributeError): # log.log("junk error", "BART") - # def test_topic_and_status_set_correctly(): # sim_id = "543" @@ -118,5 +112,3 @@ # assert message['source'] == application_id # assert topic == expected_topic # assert message['processStatus'] == "RUNNING" - - diff --git a/gridappsd-python-lib/tests/test_logging_integration.py b/gridappsd-python-lib/tests/test_logging_integration.py index adb1e2c..9e8c375 100644 --- a/gridappsd-python-lib/tests/test_logging_integration.py +++ b/gridappsd-python-lib/tests/test_logging_integration.py @@ -7,7 +7,6 @@ # from gridappsd import GridAPPSD, topics as t # from gridappsd.loghandler import Logger - # @pytest.fixture # def logger_and_gridapspd(gridappsd_client) -> (Logger, GridAPPSD): @@ -17,7 +16,6 @@ # logger = None - # @mock.patch.dict(os.environ, # dict(GRIDAPPSD_APPLICATION_ID='sample_app', # GRIDAPPSD_APPLICATION_STATUS='RUNNING')) @@ -53,7 +51,6 @@ # assert expected[1] == x['log_message'] # assert expected[2] == x['log_level'] - # SIMULATION_ID='54321' # #TODO Ask about loging api for simulations. diff --git a/gridappsd-python-lib/tests/test_simulation.py b/gridappsd-python-lib/tests/test_simulation.py index 5be2be8..b6c9365 100644 --- a/gridappsd-python-lib/tests/test_simulation.py +++ b/gridappsd-python-lib/tests/test_simulation.py @@ -14,14 +14,12 @@ # # The directory containing this file # HERE = os.path.dirname(__file__) - # def base_config(): # data = {"power_system_config":{"SubGeographicalRegion_name":"_ABEB635F-729D-24BF-B8A4-E2EF268D8B9E","GeographicalRegion_name":"_73C512BD-7249-4F50-50DA-D93849B89C43","Line_name":"_49AD8E07-3BF9-A4E2-CB8F-C3722F837B62"},"simulation_config":{"power_flow_solver_method":"NR","duration":120,"simulation_name":"ieee13nodeckt","simulator":"GridLAB-D","start_time":1605418946,"run_realtime":False,"simulation_output":{},"model_creation_config":{"load_scaling_factor":1.0,"triplex":"y","encoding":"u","system_frequency":60,"voltage_multiplier":1.0,"power_unit_conversion":1.0,"unique_names":"y","schedule_name":"ieeezipload","z_fraction":0.0,"i_fraction":1.0,"p_fraction":0.0,"randomize_zipload_fractions":False,"use_houses":False},"simulation_broker_port":51044,"simulation_broker_location":"127.0.0.1"},"application_config":{"applications":[]},"service_configs":[],"test_config":{"randomNum":{"seed":{"value":185213303967438},"nextNextGaussian":0.0,"haveNextNextGaussian":False},"events":[],"testInput":True,"testOutput":True,"appId":"","testId":"1468836560","testType":"simulation_vs_expected","storeMatches":False},"simulation_request_type":"NEW"} # # with open("{HERE}/simulation_fixtures/13_node_2_min_base.json".format(HERE=HERE)) as fp: # # data = json.load(fp) # return data - # def test_simulation_no_duplicate_measurement_timestamps(gridappsd_client: GridAPPSD): # num_measurements = 0 # timestamps = set() diff --git a/info/VERSION b/info/VERSION index 17c91dc..dd41647 100644 --- a/info/VERSION +++ b/info/VERSION @@ -1 +1 @@ -2023.5.1 \ No newline at end of file +2023.5.1 diff --git a/listen_all_topics.py b/listen_all_topics.py index 5b7982d..fed4b09 100644 --- a/listen_all_topics.py +++ b/listen_all_topics.py @@ -7,6 +7,7 @@ def cb(header, msg): print(f"header: {header} message: {msg}") + g.subscribe("/topic/data", cb) houses = g.get_houses() @@ -14,4 +15,4 @@ def cb(header, msg): print(hs) while True: - sleep(0.1) \ No newline at end of file + sleep(0.1) diff --git a/make_measurement_files.py b/make_measurement_files.py index 6b24401..213bc48 100644 --- a/make_measurement_files.py +++ b/make_measurement_files.py @@ -15,19 +15,23 @@ output_queue = Queue() gapps = GridAPPSD() -with open("/repos/gridappsd-python/examples/default_run_simulation_ieee9500_final_config.yaml") as fp: +with open( + "/repos/gridappsd-python/examples/default_run_simulation_ieee9500_final_config.yaml" +) as fp: config = yaml.safe_load(fp) -sim = Simulation(gapps=gapps, - run_config=config) +sim = Simulation(gapps=gapps, run_config=config) sim_id = sim.start_simulation() + def data_output(topic, message): print(f"data_output {json.dumps(message)[:50]}") output_queue.put(message) + gapps.subscribe(simulation_output_topic(sim.simulation_id), data_output) + def write_thread(queue, filename): with open(filename, "w") as f: while True: @@ -36,6 +40,7 @@ def write_thread(queue, filename): return f.write(json.dumps(content)) + wthread = threading.Thread(target=write_thread, args=(output_queue, "messages.json")) wthread.daemon = True @@ -44,4 +49,3 @@ def write_thread(queue, filename): sim.run_loop() output_queue.put("DONE") - diff --git a/run_simulation.py b/run_simulation.py index 557510b..257679f 100644 --- a/run_simulation.py +++ b/run_simulation.py @@ -9,22 +9,44 @@ from gridappsd import GridAPPSD, topics as t from gridappsd.simulation import Simulation - logging.basicConfig(level=logging.DEBUG) _log = logging.getLogger(__name__) logging.getLogger("gridappsd.simulation").setLevel(logging.INFO) - sensor_test = { - "_9c4360bc-13ee-47f0-8a8c-0acb6c2a9930": {"class": "Breaker", "type": "A"}, # Breaker (Analog) (A) - "_9e76659c-c1e2-47d8-bd97-3a5d31c72bc1": {"class": "LoadBreakSwitch", "type": "A"}, # LoadBreakSwitch (Analog) (A) - "_8e46d152-edbe-4190-8dfc-d4322bbc6fb8": {"class": "ACLineSegment", "type": "PNV"}, # ACLineSegment (Analog) (PNV) - "_91c8096b-527e-4b17-9b60-608c2e89b0ef": {"class": "PowerTransformer", "type": "PNV"}, # PowerTransformer (Analog) (PNV) - "_7a5ce176-8185-4118-8f49-d1628692d783": {"class": "ACLineSegment", "type": "VA"}, # ACLineSegment (Analog) (VA) - "_d3fc08bf-ab76-4bba-a3c3-b8de144310f7": {"class": "PowerTransformer", "type": "VA"}, # PowerTransformer (Analog) (VA) - "_81ebcbfc-f8fe-4b7d-9735-0b00356b24dd": {"class": "Breaker", "type": "Pos"}, # Breaker (Discrete) (Pos) - "_9530188b-b0f6-4337-84ec-2fc9282740b3": {"class": "Recloser", "type": "Pos"}, # Recloser (Discrete) (Pos) + "_9c4360bc-13ee-47f0-8a8c-0acb6c2a9930": { + "class": "Breaker", + "type": "A" + }, # Breaker (Analog) (A) + "_9e76659c-c1e2-47d8-bd97-3a5d31c72bc1": { + "class": "LoadBreakSwitch", + "type": "A" + }, # LoadBreakSwitch (Analog) (A) + "_8e46d152-edbe-4190-8dfc-d4322bbc6fb8": { + "class": "ACLineSegment", + "type": "PNV" + }, # ACLineSegment (Analog) (PNV) + "_91c8096b-527e-4b17-9b60-608c2e89b0ef": { + "class": "PowerTransformer", + "type": "PNV" + }, # PowerTransformer (Analog) (PNV) + "_7a5ce176-8185-4118-8f49-d1628692d783": { + "class": "ACLineSegment", + "type": "VA" + }, # ACLineSegment (Analog) (VA) + "_d3fc08bf-ab76-4bba-a3c3-b8de144310f7": { + "class": "PowerTransformer", + "type": "VA" + }, # PowerTransformer (Analog) (VA) + "_81ebcbfc-f8fe-4b7d-9735-0b00356b24dd": { + "class": "Breaker", + "type": "Pos" + }, # Breaker (Discrete) (Pos) + "_9530188b-b0f6-4337-84ec-2fc9282740b3": { + "class": "Recloser", + "type": "Pos" + }, # Recloser (Discrete) (Pos) } config_file = "history_config.json" @@ -51,7 +73,8 @@ os.environ['GRIDAPPSD_ADDRESS'] = 'gridappsd' gapps = GridAPPSD(goss_log_level=logging.INFO) except ConnectFailedException: - print("Failed to connect, possible system is not running or login invalid!") + print( + "Failed to connect, possible system is not running or login invalid!") sys.exit() if not gapps.connected: @@ -86,8 +109,10 @@ def onmeasurment(sim, timestamp, measurements): # # v['class'] = sensor_test[k]['class'] # # v['type'] = sensor_test[k]['type'] # measurement_output.append(v) - _log.info("{timestamp} publish number: {publish_number}".format(publish_number=publish_number, - timestamp=timestamp)) + _log.info("{timestamp} publish number: {publish_number}".format( + publish_number=publish_number, timestamp=timestamp)) + + # fd.write(f"{json.dumps(measurements)}\n") @@ -111,8 +136,9 @@ def on_simulated_output(header, message): global sim_publish_number sim_publish_number += 1 timestamp = message['message']['timestamp'] - _log.info("{timestamp} simulation publish number: {sim_publish_number} timestamp: {timestamp}".format(timestamp=timestamp, - sim_publish_number=sim_publish_number)) + _log.info( + "{timestamp} simulation publish number: {sim_publish_number} timestamp: {timestamp}" + .format(timestamp=timestamp, sim_publish_number=sim_publish_number)) # print('SIMULATED MESSAGE IS HERE!') # measurements = message['message']['measurements'] # timestamp = message['message']['timestamp'] @@ -129,8 +155,10 @@ def on_simulated_output(header, message): sim.add_ontimestep_callback(ontimestep) sim.add_oncomplete_callback(onfinishsimulation) sim.start_simulation() -read_topic = t.service_output_topic("gridappsd-sensor-simulator", sim.simulation_id) -_log.debug("Reading topic for sensor output {read_topic}".format(read_topic=read_topic)) +read_topic = t.service_output_topic("gridappsd-sensor-simulator", + sim.simulation_id) +_log.debug("Reading topic for sensor output {read_topic}".format( + read_topic=read_topic)) gapps.subscribe(read_topic, on_simulated_output) try: diff --git a/scripts/create_local_version.sh b/scripts/create_local_version.sh index 8e4bcfe..6e37c22 100755 --- a/scripts/create_local_version.sh +++ b/scripts/create_local_version.sh @@ -1,6 +1,6 @@ #!/bin/sh # Usus dunamai to determine a semver compatible version for the current state of the project -# Useefull when building wheels in CI/CD on branches or merge requests, +# Useefull when building wheels in CI/CD on branches or merge requests, # without possibly overwriting released versions (of certain tag) # Used to run in CI/CD, as it will modify both pyproject.toml's and python files (by setting the right string in `__version__=..`) set -x diff --git a/scripts/projects.sh b/scripts/projects.sh index 531e65c..94c9503 100644 --- a/scripts/projects.sh +++ b/scripts/projects.sh @@ -1,4 +1,4 @@ #!/bin/sh # all python packages, in topological order -PROJECTS='gridappsd-field-bus-lib gridappsd-python-lib' \ No newline at end of file +PROJECTS='gridappsd-field-bus-lib gridappsd-python-lib' diff --git a/scripts/replace_path_deps.sh b/scripts/replace_path_deps.sh index 0795803..48663d0 100755 --- a/scripts/replace_path_deps.sh +++ b/scripts/replace_path_deps.sh @@ -44,7 +44,7 @@ for WHEELFILE in $WHEELFILES do rm -rf /tmp/version_update mkdir -p /tmp/version_update - unzip -d /tmp/version_update $curdir/$WHEELFILE + unzip -d /tmp/version_update $curdir/$WHEELFILE cd /tmp/version_update # Replace the path dependencies (which are prefixed with '@') # with compatible version to the current monorepo, but at least at the current one. From 9a3749d3bbe1ba21ffec9ac0ccdf5e38716585bc Mon Sep 17 00:00:00 2001 From: afisher1 <4552674+afisher1@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:45:47 -0700 Subject: [PATCH 18/26] adding automatic pause option to SimulationArgs --- gridappsd-python-lib/gridappsd/simulation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gridappsd-python-lib/gridappsd/simulation.py b/gridappsd-python-lib/gridappsd/simulation.py index 8ff94b0..11f15e3 100644 --- a/gridappsd-python-lib/gridappsd/simulation.py +++ b/gridappsd-python-lib/gridappsd/simulation.py @@ -56,6 +56,7 @@ class SimulationArgs(ConfigBase): timestep_frequency: str = "1000" timestep_increment: str = "1000" run_realtime: bool = True + pause_after_measurements: bool = False simulation_name: str = "ieee13nodeckt" power_flow_solver_method: str = "NR" model_creation_config: ModelCreationConfig = __default_model_creation_config__ From 461b51a3bbc5aea13ac9c60b987e6dd6a4544ed2 Mon Sep 17 00:00:00 2001 From: afisher1 <4552674+afisher1@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:09:58 -0700 Subject: [PATCH 19/26] fix for issue#145 --- gridappsd-python-lib/gridappsd/simulation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gridappsd-python-lib/gridappsd/simulation.py b/gridappsd-python-lib/gridappsd/simulation.py index 8ff94b0..2c074ad 100644 --- a/gridappsd-python-lib/gridappsd/simulation.py +++ b/gridappsd-python-lib/gridappsd/simulation.py @@ -122,8 +122,12 @@ class Simulation: def __init__(self, gapps: 'GridAPPSD', run_config: Union[Dict, SimulationConfig]): assert type(gapps).__name__ == 'GridAPPSD', "Must be an instance of GridAPPSD" - - self._run_config = run_config + if isinstance(run_config, SimulationConfig): + self._run_config = run_config.asdict() + elif isinstance(run_config, dict): + self._run_config = run_config + else: + raise TypeError("run_config must be a dictionary or a SimulationConfig") # if isinstance(run_config, SimulationConfig): # self._run_config = run_config # else: From 1cc3bb45d29b5327613ed91487ea506027daf465 Mon Sep 17 00:00:00 2001 From: afisher1 <4552674+afisher1@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:45:36 -0700 Subject: [PATCH 20/26] exposing full json package functionality in json_extension.py --- .../gridappsd/json_extension.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/gridappsd-python-lib/gridappsd/json_extension.py b/gridappsd-python-lib/gridappsd/json_extension.py index 81d7343..76bb2eb 100644 --- a/gridappsd-python-lib/gridappsd/json_extension.py +++ b/gridappsd-python-lib/gridappsd/json_extension.py @@ -34,20 +34,28 @@ def default(self, obj: Any) -> Any: return rv -def dump(data: Any, fo: TextIO): - rv = _json.dump(data, fo, cls=JsonEncoderExtension) - - -def dumps(data: Any) -> str: - rv = _json.dumps(data, cls=JsonEncoderExtension) +def dump(data: Any, fo: TextIO, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, + separators=None, default=None, sort_keys=False, **kw): + rv = _json.dump(data, fo, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, + allow_nan=allow_nan, cls=JsonEncoderExtension, indent=indent, separators=separators, + default=default, sort_keys=sort_keys, **kw) + + +def dumps(data: Any, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, + separators=None, default=None, sort_keys=False, **kw) -> str: + rv = _json.dumps(data, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, + allow_nan=allow_nan, cls=JsonEncoderExtension, indent=indent, separators=separators, + default=default, sort_keys=sort_keys, **kw) return rv -def load(fo: TextIO) -> Any: - rv = _json.load(fo, object_hook=jsonDecoderExtension) +def load(fo: TextIO, *, cls=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) -> Any: + rv = _json.load(fo, cls=cls, object_hook=jsonDecoderExtension, parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) return rv -def loads(data: str) -> Any: - rv = _json.loads(data, object_hook=jsonDecoderExtension) +def loads(data: str, *, cls=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) -> Any: + rv = _json.loads(data, cls=cls, object_hook=jsonDecoderExtension, parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) return rv From 70a59112e15499a777a661ddb0d1822234b55da2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:48:54 +0000 Subject: [PATCH 21/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../gridappsd/json_extension.py | 98 +++++++++++++++---- 1 file changed, 80 insertions(+), 18 deletions(-) diff --git a/gridappsd-python-lib/gridappsd/json_extension.py b/gridappsd-python-lib/gridappsd/json_extension.py index 76bb2eb..692d0c6 100644 --- a/gridappsd-python-lib/gridappsd/json_extension.py +++ b/gridappsd-python-lib/gridappsd/json_extension.py @@ -34,28 +34,90 @@ def default(self, obj: Any) -> Any: return rv -def dump(data: Any, fo: TextIO, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, - separators=None, default=None, sort_keys=False, **kw): - rv = _json.dump(data, fo, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, - allow_nan=allow_nan, cls=JsonEncoderExtension, indent=indent, separators=separators, - default=default, sort_keys=sort_keys, **kw) - - -def dumps(data: Any, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, - separators=None, default=None, sort_keys=False, **kw) -> str: - rv = _json.dumps(data, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, - allow_nan=allow_nan, cls=JsonEncoderExtension, indent=indent, separators=separators, - default=default, sort_keys=sort_keys, **kw) +def dump(data: Any, + fo: TextIO, + *, + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + default=None, + sort_keys=False, + **kw): + rv = _json.dump(data, + fo, + skipkeys=skipkeys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + cls=JsonEncoderExtension, + indent=indent, + separators=separators, + default=default, + sort_keys=sort_keys, + **kw) + + +def dumps(data: Any, + *, + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + default=None, + sort_keys=False, + **kw) -> str: + rv = _json.dumps(data, + skipkeys=skipkeys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + cls=JsonEncoderExtension, + indent=indent, + separators=separators, + default=default, + sort_keys=sort_keys, + **kw) return rv -def load(fo: TextIO, *, cls=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) -> Any: - rv = _json.load(fo, cls=cls, object_hook=jsonDecoderExtension, parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) +def load(fo: TextIO, + *, + cls=None, + parse_float=None, + parse_int=None, + parse_constant=None, + object_pairs_hook=None, + **kw) -> Any: + rv = _json.load(fo, + cls=cls, + object_hook=jsonDecoderExtension, + parse_float=parse_float, + parse_int=parse_int, + parse_constant=parse_constant, + object_pairs_hook=object_pairs_hook, + **kw) return rv -def loads(data: str, *, cls=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) -> Any: - rv = _json.loads(data, cls=cls, object_hook=jsonDecoderExtension, parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) +def loads(data: str, + *, + cls=None, + parse_float=None, + parse_int=None, + parse_constant=None, + object_pairs_hook=None, + **kw) -> Any: + rv = _json.loads(data, + cls=cls, + object_hook=jsonDecoderExtension, + parse_float=parse_float, + parse_int=parse_int, + parse_constant=parse_constant, + object_pairs_hook=object_pairs_hook, + **kw) return rv From 15b11b44f50128b75cac0a209e75f9dea50013c6 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Wed, 1 May 2024 15:58:20 -0700 Subject: [PATCH 22/26] added logs --- .../gridappsd/field_interface/context_manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py index b8d89c9..2c9a9fc 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py @@ -44,11 +44,11 @@ def __init__(self, 'modelId': downstream_message_bus_def.id} feeder_dict = None while feeder_dict is None: - self.ot_connection.get_logger().info(f"Requesting topology for {self.__class__}") + self.ot_connection.get_logger().debug(f"Requesting topology for {self.__class__}") response = self.ot_connection.get_response(REQUEST_FIELD, request, timeout=10) if 'data' in response: feeder_dict = response['data'] - self.ot_connection.send_status("******RCVD FEEDER *************", '/topic/goss.gridappsd.platform.log', 'DEBUG') + self.ot_connection.get_logger().debug("Topology received at Feeder Area Context Manager") else: time.sleep(5) super().__init__(upstream_message_bus_def, downstream_message_bus_def, @@ -65,6 +65,7 @@ def __init__(self, self.neighbouring_agents = {} self.upstream_agents = {} self.downstream_agents = {} + self.ot_connection.get_logger().info("Feeder Area Context Manager Created") def on_request(self, message_bus, headers: Dict, message): @@ -122,6 +123,7 @@ def __init__(self, self.registered_agents = {} self.registered_agents[self.agent_id] = self.get_registration_details() + self.ot_connection.get_logger().info("Switch Area "+self.agent_id+" Context Manager Created") def on_request(self, message_bus, headers: Dict, message): @@ -182,6 +184,7 @@ def __init__(self, self.registered_agents = {} self.registered_agents[self.agent_id] = self.get_registration_details() + self.ot_connection.get_logger().info("Secondary Area "+self.agent_id+" Context Manager Created") def on_request(self, message_bus, headers: Dict, message): From 874cff2191f56f0ad2618186cb6221986c6898b9 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Wed, 1 May 2024 16:14:11 -0700 Subject: [PATCH 23/26] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3eacaec..c58e615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2024.1.4a0" +version = "2024.4.0a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From 1100adb8ae67c9006e5bb1446544232ebd49738d Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Mon, 13 May 2024 12:53:41 -0700 Subject: [PATCH 24/26] adding context in init --- .../gridappsd/field_interface/context_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py index 4b29878..6ad6997 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context_manager.py @@ -54,7 +54,7 @@ def __init__(self, #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - self.context = None + self.context = {'data':feeder_dict} self.registered_agents = {} self.registered_agents[self.agent_id] = self.get_registration_details() @@ -113,7 +113,7 @@ def __init__(self, #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - self.context = None + self.context = {'data':switch_area_dict} self.registered_agents = {} self.registered_agents[self.agent_id] = self.get_registration_details() @@ -172,7 +172,7 @@ def __init__(self, #Override agent_id to a static value self.agent_id = downstream_message_bus_def.id + '.context_manager' - self.context = None + self.context = {'data':secondary_area_dict} self.registered_agents = {} self.registered_agents[self.agent_id] = self.get_registration_details() From 3246b8ce664880ab79b5cef494788f24eea50259 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Mon, 13 May 2024 13:13:57 -0700 Subject: [PATCH 25/26] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c58e615..8a2758a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2024.4.0a0" +version = "2024.4.1a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", From aebb7753560e205077a9ffa5f346c6d8c56bf601 Mon Sep 17 00:00:00 2001 From: Tonya Martin Date: Tue, 9 Jul 2024 10:17:50 -0700 Subject: [PATCH 26/26] Update pyproject.toml for release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a2758a..bcbe198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python-workspace" -version = "2024.4.1a0" +version = "2024.6.0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>",