diff --git a/SoftLayer/API.py b/SoftLayer/API.py index 29c5ac689..4d2918611 100644 --- a/SoftLayer/API.py +++ b/SoftLayer/API.py @@ -9,11 +9,12 @@ import time import warnings +import concurrent.futures as cf import json import logging +import math import requests - from SoftLayer import auth as slauth from SoftLayer import config from SoftLayer import consts @@ -289,13 +290,6 @@ def call(self, service, method, *args, **kwargs): request.verify = kwargs.get('verify') if self.auth: - extra_headers = self.auth.get_headers() - if extra_headers: - warnings.warn("auth.get_headers() is deprecated and will be " - "removed in the next major version", - DeprecationWarning) - request.headers.update(extra_headers) - request = self.auth.get_request(request) request.headers.update(kwargs.get('headers', {})) @@ -352,6 +346,42 @@ def iter_call(self, service, method, *args, **kwargs): offset += limit + def cf_call(self, service, method, *args, **kwargs): + """Uses threads to iterate through API calls. + + :param service: the name of the SoftLayer API service + :param method: the method to call on the service + :param integer limit: result size for each API call (defaults to 100) + :param \\*args: same optional arguments that ``Service.call`` takes + :param \\*\\*kwargs: same optional keyword arguments that ``Service.call`` takes + """ + limit = kwargs.pop('limit', 100) + offset = kwargs.pop('offset', 0) + + if limit <= 0: + raise AttributeError("Limit size should be greater than zero.") + # This initial API call is to determine how many API calls we need to make after this first one. + first_call = self.call(service, method, offset=offset, limit=limit, *args, **kwargs) + + # This was not a list result, just return it. + if not isinstance(first_call, transports.SoftLayerListResult): + return first_call + # How many more API calls we have to make + api_calls = math.ceil((first_call.total_count - limit) / limit) + + def this_api(offset): + """Used to easily call executor.map() on this fuction""" + return self.call(service, method, offset=offset, limit=limit, *args, **kwargs) + + with cf.ThreadPoolExecutor(max_workers=10) as executor: + future_results = {} + offset_map = [x * limit for x in range(1, api_calls)] + future_results = list(executor.map(this_api, offset_map)) + # Append the results in the order they were called + for call_result in future_results: + first_call = first_call + call_result + return first_call + def __repr__(self): return "Client(transport=%r, auth=%r)" % (self.transport, self.auth) diff --git a/SoftLayer/CLI/vlan/list.py b/SoftLayer/CLI/vlan/list.py index c06b64602..ed55561db 100644 --- a/SoftLayer/CLI/vlan/list.py +++ b/SoftLayer/CLI/vlan/list.py @@ -25,17 +25,13 @@ @click.command(cls=SoftLayer.CLI.command.SLCommand, ) -@click.option('--sortby', - help='Column to sort by', - type=click.Choice(COLUMNS)) +@click.option('--sortby', help='Column to sort by', type=click.Choice(COLUMNS)) @click.option('--datacenter', '-d', help='Filter by datacenter shortname (sng01, dal05, ...)') @click.option('--number', '-n', help='Filter by VLAN number') @click.option('--name', help='Filter by VLAN name') -@click.option('--limit', '-l', - help='How many results to get in one api call, default is 100', - default=100, - show_default=True) +@click.option('--limit', '-l', default=100, show_default=True, + help='How many results to get in one api call, default is 100') @environment.pass_env def cli(env, sortby, datacenter, number, name, limit): """List VLANs. diff --git a/SoftLayer/managers/network.py b/SoftLayer/managers/network.py index 11554beee..201864606 100644 --- a/SoftLayer/managers/network.py +++ b/SoftLayer/managers/network.py @@ -515,46 +515,37 @@ def list_subnets(self, identifier=None, datacenter=None, version=0, kwargs['iter'] = True return self.client.call('Account', 'getSubnets', **kwargs) - def list_vlans(self, datacenter=None, vlan_number=None, name=None, limit=100, **kwargs): + def list_vlans(self, datacenter=None, vlan_number=None, name=None, limit=100, mask=None, _filter=None): """Display a list of all VLANs on the account. This provides a quick overview of all VLANs including information about data center residence and the number of devices attached. - :param string datacenter: If specified, the list will only contain - VLANs in the specified data center. - :param int vlan_number: If specified, the list will only contain the - VLAN matching this VLAN number. - :param int name: If specified, the list will only contain the - VLAN matching this VLAN name. + :param string datacenter: If specified, the list will only contain VLANs in the specified data center. + :param int vlan_number: If specified, the list will only contain the VLAN matching this VLAN number. + :param int name: If specified, the list will only contain the VLAN matching this VLAN name. :param dict \\*\\*kwargs: response-level options (mask, limit, etc.) """ - _filter = utils.NestedDict(kwargs.get('filter') or {}) + _filter = utils.NestedDict(_filter or {}) _filter['networkVlans']['id'] = utils.query_filter_orderby() if vlan_number: - _filter['networkVlans']['vlanNumber'] = ( - utils.query_filter(vlan_number)) + _filter['networkVlans']['vlanNumber'] = utils.query_filter(vlan_number) if name: _filter['networkVlans']['name'] = utils.query_filter(name) if datacenter: - _filter['networkVlans']['primaryRouter']['datacenter']['name'] = ( - utils.query_filter(datacenter)) - - kwargs['filter'] = _filter.to_dict() + _filter['networkVlans']['primaryRouter']['datacenter']['name'] = utils.query_filter(datacenter) - if 'mask' not in kwargs: - kwargs['mask'] = DEFAULT_VLAN_MASK + if mask is None: + mask = DEFAULT_VLAN_MASK - kwargs['iter'] = True - if limit > 0: - return self.account.getNetworkVlans(mask=kwargs['mask'], filter=_filter.to_dict(), limit=limit, iter=True) - else: - return self.account.getNetworkVlans(mask=kwargs['mask'], filter=_filter.to_dict(), iter=True) + # cf_call uses threads to get all results. + return self.client.cf_call('SoftLayer_Account', 'getNetworkVlans', + mask=mask, filter=_filter.to_dict(), limit=limit) def list_securitygroups(self, **kwargs): """List security groups.""" diff --git a/tests/CLI/modules/vlan_tests.py b/tests/CLI/modules/vlan_tests.py index 4021aa2e7..773a67320 100644 --- a/tests/CLI/modules/vlan_tests.py +++ b/tests/CLI/modules/vlan_tests.py @@ -147,9 +147,15 @@ def test_vlan_list_get_pod_with_closed_announcement(self, ngb_mock): ngb_mock.return_value = True vlan_mock = self.set_mock('SoftLayer_Account', 'getNetworkVlans') gpods_mock = self.set_mock('SoftLayer_Network_Pod', 'getAllObjects') - vlan_mock.return_value = {'primaryRouter': {'fullyQualifiedDomainName': 'bcr01a.fra02.softlayer.com', - 'id': 462912}, - 'tagReferences': ''} + vlan_mock.return_value = [ + { + "primaryRouter": { + "fullyQualifiedDomainName": "bcr01a.fra02.softlayer.com", + "id": 462912 + }, + "tagReferences": "" + } + ] gpods_mock.return_value = [{'backendRouterId': 462912, 'backendRouterName': 'bcr01a.fra02', 'datacenterId': 449506, @@ -166,9 +172,15 @@ def test_vlan_list_get_pod_with_closed_announcement_no_closure(self, ngb_mock): ngb_mock.return_value = True vlan_mock = self.set_mock('SoftLayer_Account', 'getNetworkVlans') gpods_mock = self.set_mock('SoftLayer_Network_Pod', 'getAllObjects') - vlan_mock.return_value = {'primaryRouter': {'fullyQualifiedDomainName': 'bcr01a.fra02.softlayer.com', - 'id': 462912}, - 'tagReferences': ''} + vlan_mock.return_value = [ + { + "primaryRouter": { + "fullyQualifiedDomainName": "bcr01a.fra02.softlayer.com", + "id": 462912 + }, + "tagReferences": "" + } + ] gpods_mock.return_value = [{'backendRouterId': 462912, 'backendRouterName': 'bcr01a.fra02', 'datacenterId': 449506,