diff --git a/.editorconfig b/.editorconfig index dca80abd4..ab2b975e8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,8 +14,6 @@ charset = utf-8 trim_trailing_whitespace = true # A file must end with an empty line - this is good for version control systems insert_final_newline = true -# A line should not have more than this amount of chars (not supported by all plugins) -max_line_length = 100 [*.{py,md,txt}] indent_size = 4 diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 0a83af8de..ab16ca08b 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -965,7 +965,7 @@ msgid "Install Up Next add-on" msgstr "" msgctxt "#30235" -msgid "[WIP] Send/Receive progress and watched status of the videos (works only with the main profile)" +msgid "[WIP] Synchronize the watched status of the videos with Netflix" msgstr "" msgctxt "#30236" diff --git a/resources/lib/api/website.py b/resources/lib/api/website.py index de0b854b7..ff57417da 100644 --- a/resources/lib/api/website.py +++ b/resources/lib/api/website.py @@ -47,7 +47,12 @@ # 'ichnaea_log': 'models/serverDefs/data/ICHNAEA_ROOT', can be for XSS attacks? 'api_endpoint_root_url': 'models/serverDefs/data/API_ROOT', 'api_endpoint_url': 'models/playerModel/data/config/ui/initParams/apiUrl', - 'request_id': 'models/serverDefs/data/requestId' + 'request_id': 'models/serverDefs/data/requestId', + 'asset_core': 'models/playerModel/data/config/core/assets/core', + 'ui_version': 'models/playerModel/data/config/ui/initParams/uiVersion', + 'browser_info_version': 'models/browserInfo/data/version', + 'browser_info_os_name': 'models/browserInfo/data/os/name', + 'browser_info_os_version': 'models/browserInfo/data/os/version', } PAGE_ITEM_ERROR_CODE = 'models/flow/data/fields/errorCode/value' diff --git a/resources/lib/kodi/context_menu.py b/resources/lib/kodi/context_menu.py index e43f899e5..8cf6775c1 100644 --- a/resources/lib/kodi/context_menu.py +++ b/resources/lib/kodi/context_menu.py @@ -102,7 +102,7 @@ def generate_context_menu_items(videoid): if videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]: # Add menu to allow change manually the watched status when progress manager is enabled - if g.ADDON.getSettingBool('ProgressManager_enabled') and g.LOCAL_DB.get_profile_config('isAccountOwner', False): + if g.ADDON.getSettingBool('ProgressManager_enabled'): items.insert(0, _ctx_item('change_watched_status', videoid)) return items diff --git a/resources/lib/kodi/infolabels.py b/resources/lib/kodi/infolabels.py index 699bcd019..5410f5104 100644 --- a/resources/lib/kodi/infolabels.py +++ b/resources/lib/kodi/infolabels.py @@ -351,10 +351,7 @@ def _colorize_title(text, color, remove_color=False): def _set_progress_status(list_item, video_data, infos): """Check and set progress status (watched and resume)""" - if not g.ADDON.getSettingBool('ProgressManager_enabled') or \ - not g.LOCAL_DB.get_profile_config('isAccountOwner', False): - # Currently due to a unknown problem, it is not possible to communicate MSL data to the right selected - # profile other than the owner profile + if not g.ADDON.getSettingBool('ProgressManager_enabled'): return video_id = video_data['summary']['id'] @@ -368,11 +365,10 @@ def _set_progress_status(list_item, video_data, infos): # seem not respect really if a video is watched to the end or this tag have other purposes # to now, the only way to know if a video is watched is compare the bookmarkPosition with creditsOffset value - if not video_data.get('creditsOffset'): - # NOTE shakti 'creditsOffset' tag not exists on video type 'movie', - # then simulate the default Kodi playcount behaviour (playcountminimumpercent) - watched_threshold = video_data['runtime'] - (video_data['runtime'] / 100 * 90) - else: + # NOTE shakti 'creditsOffset' tag not exists on video type 'movie', + # then simulate the default Kodi playcount behaviour (playcountminimumpercent) + watched_threshold = video_data['runtime'] / 100 * 90 + if video_data.get('creditsOffset') and video_data['creditsOffset'] < watched_threshold: watched_threshold = video_data['creditsOffset'] # NOTE shakti 'bookmarkPosition' tag when it is not set have -1 value diff --git a/resources/lib/navigation/player.py b/resources/lib/navigation/player.py index ceabb9e0a..d7cbc9755 100644 --- a/resources/lib/navigation/player.py +++ b/resources/lib/navigation/player.py @@ -89,10 +89,9 @@ def play(videoid): return if index_selected == 1: resume_position = None - elif g.ADDON.getSettingBool('ProgressManager_enabled') and g.LOCAL_DB.get_profile_config('isAccountOwner', False): + elif (g.ADDON.getSettingBool('ProgressManager_enabled') and + videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]): # To now we have this limits: - # - enabled only if the owner profile is used. Currently due to a unknown problem, - # it is not possible to communicate MSL data to the right selected profile # - enabled only with items played inside the addon then not Kodi library, need impl. JSON-RPC lib update code event_data = _get_event_data(videoid) event_data['videoid'] = videoid.to_dict() diff --git a/resources/lib/services/msl/android_crypto.py b/resources/lib/services/msl/android_crypto.py index 0c2aabb41..18f551e90 100644 --- a/resources/lib/services/msl/android_crypto.py +++ b/resources/lib/services/msl/android_crypto.py @@ -23,34 +23,27 @@ class AndroidMSLCrypto(MSLBaseCrypto): """Crypto handler for Android platforms""" - def __init__(self, msl_data=None): # pylint: disable=super-on-old-class - # pylint: disable=broad-except + def __init__(self): + super(AndroidMSLCrypto, self).__init__() + self.crypto_session = None + self.keyset_id = None + self.key_id = None + self.hmac_key_id = None try: self.crypto_session = xbmcdrm.CryptoSession( 'edef8ba9-79d6-4ace-a3c8-27dcd51d21ed', 'AES/CBC/NoPadding', 'HmacSHA256') common.debug('Widevine CryptoSession successful constructed') - except Exception: + except Exception: # pylint: disable=broad-except import traceback common.error(traceback.format_exc()) raise MSLError('Failed to construct Widevine CryptoSession') - try: - super(AndroidMSLCrypto, self).__init__(msl_data) - self.keyset_id = base64.standard_b64decode(msl_data['key_set_id']) - self.key_id = base64.standard_b64decode(msl_data['key_id']) - self.hmac_key_id = base64.standard_b64decode(msl_data['hmac_key_id']) - self.crypto_session.RestoreKeys(self.keyset_id) - except Exception: - self.keyset_id = None - self.key_id = None - self.hmac_key_id = None - drm_info = { 'version': self.crypto_session.GetPropertyString('version'), 'system_id': self.crypto_session.GetPropertyString('systemId'), # 'device_unique_id': self.crypto_session.GetPropertyByteArray('deviceUniqueId') - 'hdcp_level': self.crypto_session.GetPropertyString('hdcpLevel'), - 'hdcp_level_max': self.crypto_session.GetPropertyString('maxHdcpLevel'), + 'hdcp_level': self.crypto_session.GetPropertyString('hdcpLevel'), #WTF perchè non viene fuori??? serve volontario test + 'hdcp_level_max': self.crypto_session.GetPropertyString('maxHdcpLevel'), #WTF perchè non viene fuori??? serve volontario test 'security_level': self.crypto_session.GetPropertyString('securityLevel') } @@ -71,10 +64,18 @@ def __init__(self, msl_data=None): # pylint: disable=super-on-old-class else: common.warn('Widevine CryptoSession system id not obtained!') common.debug('Widevine CryptoSession security level: {}', drm_info['security_level']) - common.debug('Widevine CryptoSession current hdcp level', drm_info['hdcp_level']) - common.debug('Widevine CryptoSession max hdcp level supported', drm_info['hdcp_level_max']) + common.debug('Widevine CryptoSession current hdcp level: {}', drm_info['hdcp_level']) + common.debug('Widevine CryptoSession max hdcp level supported: {}', drm_info['hdcp_level_max']) common.debug('Widevine CryptoSession algorithms: {}', self.crypto_session.GetPropertyString('algorithms')) + def load_crypto_session(self, msl_data=None): + if not msl_data: + return + self.keyset_id = base64.standard_b64decode(msl_data['key_set_id']) + self.key_id = base64.standard_b64decode(msl_data['key_id']) + self.hmac_key_id = base64.standard_b64decode(msl_data['hmac_key_id']) + self.crypto_session.RestoreKeys(self.keyset_id) + def __del__(self): self.crypto_session = None diff --git a/resources/lib/services/msl/base_crypto.py b/resources/lib/services/msl/base_crypto.py index 29511fdb7..d24427483 100644 --- a/resources/lib/services/msl/base_crypto.py +++ b/resources/lib/services/msl/base_crypto.py @@ -11,6 +11,7 @@ import json import base64 +import time import resources.lib.common as common @@ -18,18 +19,25 @@ class MSLBaseCrypto(object): """Common base class for MSL crypto operations. Handles mastertoken and sequence number""" - # pylint: disable=too-few-public-methods - def __init__(self, msl_data=None): + + def __init__(self): + self._msl_data = None + self.mastertoken = None + self.serial_number = None + self.sequence_number = None + self.renewal_window = None + self.expiration = None + + def load_msl_data(self, msl_data=None): + self._msl_data = msl_data if msl_data else {} if msl_data: - self._set_mastertoken(msl_data['tokens']['mastertoken']) - else: - self.mastertoken = None + self.set_mastertoken(msl_data['tokens']['mastertoken']) def compare_mastertoken(self, mastertoken): """Check if the new mastertoken is different from current due to renew""" if not self._mastertoken_is_newer_that(mastertoken): common.debug('MSL mastertoken is changed due to renew') - self._set_mastertoken(mastertoken) + self.set_mastertoken(mastertoken) self._save_msl_data() def _mastertoken_is_newer_that(self, mastertoken): @@ -48,12 +56,12 @@ def _mastertoken_is_newer_that(self, mastertoken): def parse_key_response(self, headerdata, save_to_disk): """Parse a key response and update crypto keys""" - self._set_mastertoken(headerdata['keyresponsedata']['mastertoken']) + self.set_mastertoken(headerdata['keyresponsedata']['mastertoken']) self._init_keys(headerdata['keyresponsedata']) if save_to_disk: self._save_msl_data() - def _set_mastertoken(self, mastertoken): + def set_mastertoken(self, mastertoken): """Set the mastertoken and check it for validity""" tokendata = json.loads( base64.standard_b64decode(mastertoken['tokendata'].encode('utf-8')).decode('utf-8')) @@ -65,9 +73,9 @@ def _set_mastertoken(self, mastertoken): def _save_msl_data(self): """Save crypto keys and mastertoken to disk""" - msl_data = {'tokens': {'mastertoken': self.mastertoken}} - msl_data.update(self._export_keys()) - common.save_file('msl_data.json', json.dumps(msl_data).encode('utf-8')) + self._msl_data['tokens'] = {'mastertoken': self.mastertoken} + self._msl_data.update(self._export_keys()) + common.save_file('msl_data.json', json.dumps(self._msl_data).encode('utf-8')) common.debug('Successfully saved MSL data to disk') def _init_keys(self, key_response_data): @@ -77,3 +85,32 @@ def _init_keys(self, key_response_data): def _export_keys(self): """Export crypto keys to a dict""" raise NotImplementedError + + def get_user_id_token(self, profile_guid): + """Get a valid the user id token associated to a profile guid""" + if 'user_id_tokens' in self._msl_data: + user_id_token = self._msl_data['user_id_tokens'].get(profile_guid) + if user_id_token and self.is_user_id_token_valid(user_id_token): + return user_id_token + return None + + def save_user_id_token(self, profile_guid, user_token_id): + """Save or update a user id token associated to a profile guid""" + if 'user_id_tokens' not in self._msl_data: + self._msl_data['user_id_tokens'] = { + profile_guid: user_token_id + } + else: + self._msl_data['user_id_tokens'][profile_guid] = user_token_id + self._save_msl_data() + + def clear_user_id_tokens(self): + """Clear all user id tokens""" + self._msl_data.pop('user_id_tokens', None) + self._save_msl_data() + + def is_user_id_token_valid(self, user_id_token): + """Check if user id token is not expired""" + token_data = json.loads(base64.standard_b64decode(user_id_token['tokendata'])) + # Subtract 5min as a safety measure + return (token_data['expiration'] - 300) > time.time() diff --git a/resources/lib/services/msl/default_crypto.py b/resources/lib/services/msl/default_crypto.py index fee59996c..3feff711d 100644 --- a/resources/lib/services/msl/default_crypto.py +++ b/resources/lib/services/msl/default_crypto.py @@ -10,8 +10,9 @@ """ from __future__ import absolute_import, division, unicode_literals -import json import base64 +import json + try: # Python 3 from Crypto.Random import get_random_bytes from Crypto.Hash import HMAC, SHA256 @@ -30,23 +31,29 @@ import resources.lib.common as common from .base_crypto import MSLBaseCrypto +from .exceptions import MSLError class DefaultMSLCrypto(MSLBaseCrypto): """Crypto Handler for non-Android platforms""" - def __init__(self, msl_data=None): # pylint: disable=super-on-old-class - # pylint: disable=broad-except + + def __init__(self): + super(DefaultMSLCrypto, self).__init__() + self.rsa_key = None + self.encryption_key = None + self.sign_key = None + + def load_crypto_session(self, msl_data=None): try: - super(DefaultMSLCrypto, self).__init__(msl_data) self.encryption_key = base64.standard_b64decode( msl_data['encryption_key']) self.sign_key = base64.standard_b64decode( msl_data['sign_key']) if not self.encryption_key or not self.sign_key: - raise ValueError('Missing encryption_key or sign_key') + raise MSLError('Missing encryption_key or sign_key') self.rsa_key = RSA.importKey( base64.standard_b64decode(msl_data['rsa_key'])) - except Exception: + except Exception: # pylint: disable=broad-except common.debug('Generating new RSA keys') self.rsa_key = RSA.generate(2048) self.encryption_key = None diff --git a/resources/lib/services/msl/event_tag_builder.py b/resources/lib/services/msl/event_tag_builder.py deleted file mode 100644 index 22ba6154c..000000000 --- a/resources/lib/services/msl/event_tag_builder.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) - Copyright (C) 2019 Stefano Gottardo - @CastagnaIT (original implementation module) - Build event tags values - - SPDX-License-Identifier: MIT - See LICENSES/MIT.md for more information. -""" -from __future__ import absolute_import, division, unicode_literals - -from resources.lib import common - -AUDIO_CHANNELS_CONV = {1: '1.0', 2: '2.0', 6: '5.1', 8: '7.1'} - - -def is_media_changed(previous_player_state, player_state): - """Check if there are variations in player state to avoids overhead processing""" - if not previous_player_state: - return True - # To now we do not check subtitle, because to the moment it is not implemented - if player_state['currentvideostream'] != previous_player_state['currentvideostream'] or \ - player_state['currentaudiostream'] != previous_player_state['currentaudiostream']: - return True - return False - - -def update_play_times_duration(play_times, player_state): - """Update the playTimes duration values""" - duration = player_state['elapsed_seconds'] * 1000 - play_times['total'] = duration - play_times['audio'][0]['duration'] = duration - play_times['video'][0]['duration'] = duration - - -def build_media_tag(player_state, manifest): - """Build the playTimes and the mediaId data by parsing manifest and the current player streams used""" - common.fix_locale_languages(manifest['audio_tracks']) - duration = player_state['elapsed_seconds'] * 1000 - - audio_downloadable_id, audio_track_id = _find_audio_data(player_state, manifest) - video_downloadable_id, video_track_id = _find_video_data(player_state, manifest) - # Warning 'currentsubtitle' value in player_state on Kodi 18 - # do not have proprieties like isdefault, isforced, isimpaired - # if in the future the implementation will be done it should be available only on Kodi 19 - # then for now we leave the subtitles as disabled - - text_track_id = 'T:1:1;1;NONE;0;1;' - - play_times = { - 'total': duration, - 'audio': [{ - 'downloadableId': audio_downloadable_id, - 'duration': duration - }], - 'video': [{ - 'downloadableId': video_downloadable_id, - 'duration': duration - }], - 'text': [] - } - - # Format example: "A:1:1;2;en;1;|V:2:1;2;;default;1;CE3;0;|T:1:1;1;NONE;0;1;" - media_id = '|'.join([audio_track_id, video_track_id, text_track_id]) - - return play_times, media_id - - -def _find_audio_data(player_state, manifest): - """ - Find the audio downloadable id and the audio track id - """ - language = common.convert_language_iso(player_state['currentaudiostream']['language']) - channels = AUDIO_CHANNELS_CONV[player_state['currentaudiostream']['channels']] - - for audio_track in manifest['audio_tracks']: - if audio_track['language'] == language and audio_track['channels'] == channels: - # Get the stream dict with the highest bitrate - stream = max(audio_track['streams'], key=lambda x: x['bitrate']) - return stream['downloadable_id'], audio_track['new_track_id'] - # Not found? - raise Exception('build_media_tag: unable to find audio data with language: {}, channels: {}' - .format(language, channels)) - - -def _find_video_data(player_state, manifest): - """ - Find the best match for the video downloadable id and the video track id - """ - codec = player_state['currentvideostream']['codec'] - width = player_state['currentvideostream']['width'] - height = player_state['currentvideostream']['height'] - for video_track in manifest['video_tracks']: - for stream in video_track['streams']: - if codec in stream['content_profile'] and width == stream['res_w'] and height == stream['res_h']: - return stream['downloadable_id'], video_track['new_track_id'] - # Not found? - raise Exception('build_media_tag: unable to find video data with codec: {}, width: {}, height: {}' - .format(codec, width, height)) diff --git a/resources/lib/services/msl/events_handler.py b/resources/lib/services/msl/events_handler.py index f2444e267..5013c104f 100644 --- a/resources/lib/services/msl/events_handler.py +++ b/resources/lib/services/msl/events_handler.py @@ -25,20 +25,14 @@ from resources.lib import common from resources.lib.database.db_utils import TABLE_SESSION from resources.lib.globals import g -from resources.lib.services.msl import event_tag_builder -from resources.lib.services.msl.msl_handler_base import build_request_data, ENDPOINTS +from resources.lib.services.msl import msl_utils +from resources.lib.services.msl.msl_utils import EVENT_START, EVENT_STOP, EVENT_ENGAGE, ENDPOINTS try: import Queue as queue except ImportError: # Python 3 import queue -EVENT_START = 'start' # events/start : Video starts -EVENT_STOP = 'stop' # events/stop : Video stops -EVENT_KEEP_ALIVE = 'keepAlive' # events/keepAlive : Update progress status -EVENT_ENGAGE = 'engage' # events/engage : After user interaction (before stop, on skip, on pause) -EVENT_BIND = 'bind' # events/bind : ? - class Event(object): """Object representing an event request to be processed""" @@ -123,7 +117,7 @@ def _process_event_request(self, event): 'reqName': 'events/{}'.format(event)} url = ENDPOINTS['events'] + '?' + urlencode(params).replace('%2F', '/') try: - response = self.chunked_request(url, event.request_data, g.get_esn()) + response = self.chunked_request(url, event.request_data, g.get_esn(), disable_msl_switch=False) event.set_response(response) break except Exception as exc: # pylint: disable=broad-except @@ -160,7 +154,12 @@ def add_event_to_queue(self, event_type, event_data, player_state): event_type, previous_data.get('xid')) return - event_data = build_request_data(url, self._build_event_params(event_type, event_data, player_state, manifest)) + from resources.lib.services.msl.msl_request_builder import MSLRequestBuilder + event_data = MSLRequestBuilder.build_request_data(url, + self._build_event_params(event_type, + event_data, + player_state, + manifest)) try: self.queue_events.put_nowait(Event(event_data)) except queue.Full: @@ -192,11 +191,11 @@ def _build_event_params(self, event_type, event_data, player_state, manifest): # else: # list_id = g.LOCAL_DB.get_value('last_menu_id', 'unknown') - if event_tag_builder.is_media_changed(previous_player_state, player_state): - play_times, media_id = event_tag_builder.build_media_tag(player_state, manifest) + if msl_utils.is_media_changed(previous_player_state, player_state): + play_times, media_id = msl_utils.build_media_tag(player_state, manifest) else: play_times = previous_data['playTimes'] - event_tag_builder.update_play_times_duration(play_times, player_state) + msl_utils.update_play_times_duration(play_times, player_state) media_id = previous_data['mediaId'] params = { @@ -237,5 +236,5 @@ def _build_event_params(self, event_type, event_data, player_state, manifest): def get_manifest(videoid): """Get the manifest from cache""" - cache_identifier = g.get_esn() + '_' + videoid.value + cache_identifier = g.LOCAL_DB.get_active_profile_guid() + '_' + g.get_esn() + '_' + videoid.value return g.CACHE.get(cache.CACHE_MANIFESTS, cache_identifier, False) diff --git a/resources/lib/services/msl/msl_handler.py b/resources/lib/services/msl/msl_handler.py index ec68eef53..9503fadd1 100644 --- a/resources/lib/services/msl/msl_handler.py +++ b/resources/lib/services/msl/msl_handler.py @@ -2,7 +2,7 @@ """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) Copyright (C) 2017 Trummerjo (original implementation module) - Proxy service to convert manifest and provide license data + Proxy service to convert manifest, provide license data and handle events SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. @@ -12,7 +12,6 @@ import json import time -import requests import xbmcaddon import resources.lib.cache as cache @@ -21,9 +20,9 @@ from resources.lib.globals import g from .converter import convert_to_dash from .events_handler import EventsHandler -from .msl_handler_base import MSLHandlerBase, ENDPOINTS, display_error_info, build_request_data +from .msl_requests import MSLRequests +from .msl_utils import ENDPOINTS, display_error_info from .profiles import enabled_profiles -from .request_builder import MSLRequestBuilder try: # Python 2 unicode @@ -31,39 +30,30 @@ unicode = str # pylint: disable=redefined-builtin -class MSLHandler(MSLHandlerBase): - """Handles session management and crypto for license and manifest - requests""" +class MSLHandler(object): + """Handles session management and crypto for license, manifest and event requests""" last_license_session_id = '' last_license_url = '' last_license_release_url = '' last_drm_context = '' last_playback_context = '' - session = requests.session() def __init__(self): super(MSLHandler, self).__init__() - # pylint: disable=broad-except - self.request_builder = None + self.msl_requests = None try: msl_data = json.loads(common.load_file('msl_data.json')) common.info('Loaded MSL data from disk') - except Exception: + except Exception: # pylint: disable=broad-except msl_data = None - try: - self.request_builder = MSLRequestBuilder(msl_data) - # Addon just installed, the service starts but there is no esn - if g.get_esn(): - self.check_mastertoken_validity() - - events_handler = EventsHandler(self.chunked_request) - events_handler.start() - except Exception: - import traceback - common.error(traceback.format_exc()) + + self.msl_requests = MSLRequests(msl_data) + + EventsHandler(self.msl_requests.chunked_request).start() + common.register_slot( signal=common.Signals.ESN_CHANGED, - callback=self.perform_key_handshake) + callback=self.msl_requests.perform_key_handshake) common.register_slot( signal=common.Signals.RELEASE_LICENSE, callback=self.release_license) @@ -72,8 +62,7 @@ def __init__(self): @common.time_execution(immediate=True) def load_manifest(self, viewable_id): """ - Loads the manifets for the given viewable_id and - returns a mpd-XML-Manifest + Loads the manifests for the given viewable_id and returns a mpd-XML-Manifest :param viewable_id: The id of of the viewable :return: MPD XML Manifest or False if no success @@ -88,23 +77,22 @@ def load_manifest(self, viewable_id): return self.__tranform_to_dash(manifest) def get_edge_manifest(self, viewable_id, chrome_manifest): - """Load a manifest with an EDGE ESN and replace playback_context and - drm_context""" + """Load a manifest with an EDGE ESN and replace playback_context and drm_context""" common.debug('Loading EDGE manifest') esn = g.get_edge_esn() common.debug('Switching MSL data to EDGE') - self.perform_key_handshake(esn) + self.msl_requests.perform_key_handshake(esn) manifest = self._load_manifest(viewable_id, esn) manifest['playbackContextId'] = chrome_manifest['playbackContextId'] manifest['drmContextId'] = chrome_manifest['drmContextId'] common.debug('Successfully loaded EDGE manifest') common.debug('Resetting MSL data to Chrome') - self.perform_key_handshake() + self.msl_requests.perform_key_handshake() return manifest @common.time_execution(immediate=True) def _load_manifest(self, viewable_id, esn): - cache_identifier = esn + '_' + unicode(viewable_id) + cache_identifier = g.LOCAL_DB.get_active_profile_guid() + '_' + esn + '_' + unicode(viewable_id) try: # The manifest must be requested once and maintained for its entire duration manifest = g.CACHE.get(cache.CACHE_MANIFESTS, cache_identifier, False) @@ -169,12 +157,10 @@ def _load_manifest(self, viewable_id, esn): 'preferAssistiveAudio': False } - # Get and check mastertoken validity - mt_validity = self.check_mastertoken_validity() - manifest = self.chunked_request(ENDPOINTS['manifest'], - build_request_data('/manifest', params), - esn, - mt_validity) + manifest = self.msl_requests.chunked_request(ENDPOINTS['manifest'], + self.msl_requests.build_request_data('/manifest', params), + esn, + disable_msl_switch=False) if common.is_debug_verbose(): # Save the manifest to disk as reference common.save_file('manifest.json', json.dumps(manifest).encode('utf-8')) @@ -190,6 +176,7 @@ def _load_manifest(self, viewable_id, esn): def get_license(self, challenge, sid): """ Requests and returns a license for the given challenge and sid + :param challenge: The base64 encoded challenge :param sid: The sid paired to the challenge :return: Base64 representation of the license key or False unsuccessful @@ -198,31 +185,44 @@ def get_license(self, challenge, sid): timestamp = int(time.time() * 10000) xid = str(timestamp + 1610) - params = [{ 'sessionId': sid, 'clientTime': int(timestamp / 10000), 'challengeBase64': challenge, 'xid': xid }] - - response = self.chunked_request(ENDPOINTS['license'], - build_request_data(self.last_license_url, params, 'sessionId'), - g.get_esn()) - + response = self.msl_requests.chunked_request(ENDPOINTS['license'], + self.msl_requests.build_request_data(self.last_license_url, + params, + 'sessionId'), + g.get_esn()) # This xid must be used for any future request, until playback stops g.LOCAL_DB.set_value('xid', xid, TABLE_SESSION) self.last_license_session_id = sid self.last_license_release_url = response[0]['links']['releaseLicense']['href'] + + if self.msl_requests.msl_switch_requested: + self.msl_requests.msl_switch_requested = False + self.bind_events() return response[0]['licenseResponseBase64'] + def bind_events(self): + """Bind events""" + # I don't know the real purpose of its use, it seems to be requested after the license and before starting + # playback, and only the first time after a switch, + # in the response you can also understand if the msl switch has worked + common.debug('Requesting bind events') + response = self.msl_requests.chunked_request(ENDPOINTS['events'], + self.msl_requests.build_request_data('/bind', {}), + g.get_esn(), + disable_msl_switch=False) + common.debug('Bind events response: {}', response) + @display_error_info @common.time_execution(immediate=True) def release_license(self, data=None): # pylint: disable=unused-argument - """ - Release the server license - """ - common.debug('Releasing license') + """Release the server license""" + common.debug('Requesting releasing license') params = [{ 'url': self.last_license_release_url, @@ -233,9 +233,9 @@ def release_license(self, data=None): # pylint: disable=unused-argument 'echo': 'sessionId' }] - response = self.chunked_request(ENDPOINTS['license'], - build_request_data('/bundle', params), - g.get_esn()) + response = self.msl_requests.chunked_request(ENDPOINTS['license'], + self.msl_requests.build_request_data('/bundle', params), + g.get_esn()) common.debug('License release response: {}', response) @common.time_execution(immediate=True) @@ -247,7 +247,6 @@ def __tranform_to_dash(self, manifest): def has_1080p(manifest): - """Return True if any of the video tracks in manifest have a 1080p profile - available, else False""" + """Return True if any of the video tracks in manifest have a 1080p profile available, else False""" return any(video['width'] >= 1920 for video in manifest['videoTracks'][0]['downloadables']) diff --git a/resources/lib/services/msl/msl_handler_base.py b/resources/lib/services/msl/msl_handler_base.py deleted file mode 100644 index 2570a5f80..000000000 --- a/resources/lib/services/msl/msl_handler_base.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) - Copyright (C) 2017 Trummerjo (original implementation module) - Proxy service to convert manifest and provide license data - - SPDX-License-Identifier: MIT - See LICENSES/MIT.md for more information. -""" -from __future__ import absolute_import, division, unicode_literals - -import base64 -import json -import re -import time -import zlib -from functools import wraps - -import requests - -import resources.lib.common as common -import resources.lib.kodi.ui as ui -from resources.lib.globals import g -from resources.lib.services.msl.exceptions import MSLError -from resources.lib.services.msl.request_builder import MSLRequestBuilder - -try: # Python 2 - unicode -except NameError: # Python 3 - unicode = str # pylint: disable=redefined-builtin - - -CHROME_BASE_URL = 'https://www.netflix.com/nq/msl_v1/cadmium/' -ENDPOINTS = { - 'manifest': CHROME_BASE_URL + 'pbo_manifests/%5E1.0.0/router', # "pbo_manifests/^1.0.0/router" - 'license': CHROME_BASE_URL + 'pbo_licenses/%5E1.0.0/router', - 'events': CHROME_BASE_URL + 'pbo_events/%5E1.0.0/router' -} - - -def display_error_info(func): - """Decorator that catches errors raise by the decorated function, - displays an error info dialog in the UI and reraises the error""" - # pylint: disable=missing-docstring - @wraps(func) - def error_catching_wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as exc: - ui.show_error_info(common.get_local_string(30028), unicode(exc), - unknown_error=not(unicode(exc)), - netflix_error=isinstance(exc, MSLError)) - raise - return error_catching_wrapper - - -class MSLHandlerBase(object): - """Handles session management and crypto for license, manifest and event requests""" - last_license_url = '' - last_drm_context = '' - last_playback_context = '' - session = requests.session() - - def __init__(self): - self.request_builder = None - - def check_mastertoken_validity(self): - """Return the mastertoken validity and executes a new key handshake when necessary""" - if self.request_builder.crypto.mastertoken: - time_now = time.time() - renewable = self.request_builder.crypto.renewal_window < time_now - expired = self.request_builder.crypto.expiration <= time_now - else: - renewable = False - expired = True - if expired: - if not self.request_builder.crypto.mastertoken: - common.debug('Stored MSL data not available, a new key handshake will be performed') - self.request_builder = MSLRequestBuilder() - else: - common.debug('Stored MSL data is expired, a new key handshake will be performed') - if self.perform_key_handshake(): - self.request_builder = MSLRequestBuilder(json.loads( - common.load_file('msl_data.json'))) - return self.check_mastertoken_validity() - return {'renewable': renewable, 'expired': expired} - - @display_error_info - @common.time_execution(immediate=True) - def perform_key_handshake(self, data=None): - """Perform a key handshake and initialize crypto keys""" - # pylint: disable=unused-argument - esn = data or g.get_esn() - if not esn: - common.info('Cannot perform key handshake, missing ESN') - return False - - common.debug('Performing key handshake. ESN: {}', esn) - - response = _process_json_response( - self._post(ENDPOINTS['manifest'], - self.request_builder.handshake_request(esn))) - header_data = self.request_builder.decrypt_header_data(response['headerdata'], False) - self.request_builder.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) - # Reset the user id token - self.request_builder.user_id_token = None - common.debug('Key handshake successful') - return True - - @common.time_execution(immediate=True) - def chunked_request(self, endpoint, request_data, esn, mt_validity=None): - """Do a POST request and process the chunked response""" - chunked_response = self._process_chunked_response( - self._post(endpoint, self.request_builder.msl_request(request_data, esn)), - mt_validity['renewable'] if mt_validity else None) - return chunked_response['result'] - - @common.time_execution(immediate=True) - def _post(self, endpoint, request_data): - """Execute a post request""" - common.debug('Executing POST request to {}', endpoint) - start = time.clock() - response = self.session.post(endpoint, request_data) - common.debug('Request took {}s', time.clock() - start) - common.debug('Request returned response with status {}', response.status_code) - response.raise_for_status() - return response - - # pylint: disable=unused-argument - @common.time_execution(immediate=True) - def _process_chunked_response(self, response, mt_renewable): - """Parse and decrypt an encrypted chunked response. Raise an error - if the response is plaintext json""" - try: - # if the json() does not fail we have an error because - # the expected response is a chunked json response - return _raise_if_error(response.json()) - except ValueError: - # json() failed so parse and decrypt the chunked response - common.debug('Received encrypted chunked response') - response = _parse_chunks(response.text) - # TODO: sending for the renewal request is not yet implemented - # if mt_renewable: - # # Check if mastertoken is renewed - # self.request_builder.crypto.compare_mastertoken(response['header']['mastertoken']) - - header_data = self.request_builder.decrypt_header_data( - response['header'].get('headerdata')) - - if 'useridtoken' in header_data: - # After the first call, it is possible get the 'user id token' that contains the - # user identity to use instead of 'User Authentication Data' with user credentials - self.request_builder.user_id_token = header_data['useridtoken'] - # if 'keyresponsedata' in header_data: - # common.debug('Found key handshake in response data') - # # Update current mastertoken - # self.request_builder.crypto.parse_key_response(header_data, True) - decrypted_response = _decrypt_chunks(response['payloads'], self.request_builder.crypto) - return _raise_if_error(decrypted_response) - - -def build_request_data(url, params=None, echo=''): - """Create a standard request data""" - if not params: - raise Exception('Cannot build the message without parameters') - timestamp = int(time.time() * 10000) - request_data = { - 'version': 2, - 'url': url, - 'id': timestamp, - 'languages': [g.LOCAL_DB.get_profile_config('language')], - 'params': params, - 'echo': echo - } - return request_data - - -@common.time_execution(immediate=True) -def _process_json_response(response): - """Execute a post request and expect a JSON response""" - try: - return _raise_if_error(response.json()) - except ValueError: - raise MSLError('Expected JSON response, got {}'.format(response.text)) - - -def _raise_if_error(decoded_response): - raise_error = False - # Catch a manifest/chunk error - if any(key in decoded_response for key in ['error', 'errordata']): - raise_error = True - # Catch a license error - if 'result' in decoded_response and isinstance(decoded_response.get('result'), list): - if 'error' in decoded_response['result'][0]: - raise_error = True - if raise_error: - common.error('Full MSL error information:') - common.error(json.dumps(decoded_response)) - raise MSLError(_get_error_details(decoded_response)) - return decoded_response - - -def _get_error_details(decoded_response): - # Catch a chunk error - if 'errordata' in decoded_response: - return json.loads( - base64.standard_b64decode( - decoded_response['errordata']))['errormsg'] - # Catch a manifest error - if 'error' in decoded_response: - if decoded_response['error'].get('errorDisplayMessage'): - return decoded_response['error']['errorDisplayMessage'] - # Catch a license error - if 'result' in decoded_response and isinstance(decoded_response.get('result'), list): - if 'error' in decoded_response['result'][0]: - if decoded_response['result'][0]['error'].get('errorDisplayMessage'): - return decoded_response['result'][0]['error']['errorDisplayMessage'] - return 'Unhandled error check log.' - - -@common.time_execution(immediate=True) -def _parse_chunks(message): - header = json.loads(message.split('}}')[0] + '}}') - payloads = re.split(',\"signature\":\"[0-9A-Za-z=/+]+\"}', - message.split('}}')[1]) - payloads = [x + '}' for x in payloads][:-1] - return {'header': header, 'payloads': payloads} - - -@common.time_execution(immediate=True) -def _decrypt_chunks(chunks, crypto): - decrypted_payload = '' - for chunk in chunks: - payloadchunk = json.loads(chunk) - payload = payloadchunk.get('payload') - decoded_payload = base64.standard_b64decode(payload) - encryption_envelope = json.loads(decoded_payload) - # Decrypt the text - plaintext = crypto.decrypt( - base64.standard_b64decode(encryption_envelope['iv']), - base64.standard_b64decode(encryption_envelope.get('ciphertext'))) - # unpad the plaintext - plaintext = json.loads(plaintext) - data = plaintext.get('data') - - # uncompress data if compressed - if plaintext.get('compressionalgo') == 'GZIP': - decoded_data = base64.standard_b64decode(data) - data = zlib.decompress(decoded_data, 16 + zlib.MAX_WBITS).decode('utf-8') - else: - data = base64.standard_b64decode(data).decode('utf-8') - - decrypted_payload += data - - return json.loads(decrypted_payload) diff --git a/resources/lib/services/msl/request_builder.py b/resources/lib/services/msl/msl_request_builder.py similarity index 61% rename from resources/lib/services/msl/request_builder.py rename to resources/lib/services/msl/msl_request_builder.py index 091c496fa..1c66a4ce4 100644 --- a/resources/lib/services/msl/request_builder.py +++ b/resources/lib/services/msl/msl_request_builder.py @@ -35,16 +35,30 @@ class MSLRequestBuilder(object): """Provides mechanisms to create MSL requests""" - def __init__(self, msl_data=None): + + def __init__(self): self.current_message_id = None - self.user_id_token = None self.rndm = random.SystemRandom() - self.crypto = MSLCrypto(msl_data) + self.crypto = MSLCrypto() + + @staticmethod + def build_request_data(url, params=None, echo=''): + """Create a standard request data""" + timestamp = int(time.time() * 10000) + request_data = { + 'version': 2, + 'url': url, + 'id': timestamp, + 'languages': [g.LOCAL_DB.get_profile_config('language')], + 'params': params, + 'echo': echo + } + return request_data @common.time_execution(immediate=True) - def msl_request(self, data, esn): + def msl_request(self, data, esn, auth_data): """Create an encrypted MSL request""" - return (json.dumps(self._signed_header(esn)) + + return (json.dumps(self._signed_header(esn, auth_data)) + json.dumps(self._encrypted_chunk(data, esn))) @common.time_execution(immediate=True) @@ -56,15 +70,15 @@ def handshake_request(self, esn): 'authdata': {'identity': esn}}, 'headerdata': base64.standard_b64encode( - self._headerdata(is_handshake=True).encode('utf-8')).decode('utf-8'), + self._headerdata(auth_data={}, is_handshake=True).encode('utf-8')).decode('utf-8'), 'signature': '' }, sort_keys=True) payload = json.dumps(self._encrypted_chunk(envelope_payload=False)) return header + payload @common.time_execution(immediate=True) - def _signed_header(self, esn): - encryption_envelope = self.crypto.encrypt(self._headerdata(esn=esn), esn) + def _signed_header(self, esn, auth_data): + encryption_envelope = self.crypto.encrypt(self._headerdata(auth_data=auth_data, esn=esn), esn) return { 'headerdata': base64.standard_b64encode( encryption_envelope.encode('utf-8')).decode('utf-8'), @@ -72,7 +86,7 @@ def _signed_header(self, esn): 'mastertoken': self.crypto.mastertoken, } - def _headerdata(self, esn=None, compression=None, is_handshake=False): + def _headerdata(self, auth_data, esn=None, compression=None, is_handshake=False): """ Function that generates a MSL header dict :return: The base64 encoded JSON String of the header @@ -91,7 +105,7 @@ def _headerdata(self, esn=None, compression=None, is_handshake=False): header_data['keyrequestdata'] = self.crypto.key_request_data() else: header_data['sender'] = esn - _add_auth_info(header_data, self.user_id_token) + self._add_auth_info(header_data, auth_data) return json.dumps(header_data) @@ -121,25 +135,33 @@ def decrypt_header_data(self, data, enveloped=True): return json.loads(self.crypto.decrypt(init_vector, cipher_text)) return header_data - -def _add_auth_info(header_data, user_id_token): - """User authentication identifies the application user associated with a message""" - if user_id_token and _is_useridtoken_valid(user_id_token): - # Authentication with user ID token containing the user identity - header_data['useridtoken'] = user_id_token - else: - # Authentication with the user credentials - credentials = common.get_credentials() - header_data['userauthdata'] = { - 'scheme': 'EMAIL_PASSWORD', - 'authdata': { - 'email': credentials['email'], - 'password': credentials['password'] + def _add_auth_info(self, header_data, auth_data): + """User authentication identifies the application user associated with a message""" + # Warning: the user id token contains also contains the identity of the netflix profile + # therefore it is necessary to use the right user id token for the request + if auth_data.get('user_id_token'): + if auth_data['use_switch_profile']: + # The SWITCH_PROFILE is a custom Netflix MSL user authentication scheme + # that is needed for switching profile on MSL side + # works only combined with user id token and can not be used with all endpoints + # after use it you will get user id token of the profile specified in the response + header_data['userauthdata'] = { + 'scheme': 'SWITCH_PROFILE', + 'authdata': { + 'useridtoken': auth_data['user_id_token'], + 'profileguid': g.LOCAL_DB.get_active_profile_guid() + } + } + else: + # Authentication with user ID token containing the user identity (netflix profile) + header_data['useridtoken'] = auth_data['user_id_token'] + else: + # Authentication with the user credentials + credentials = common.get_credentials() + header_data['userauthdata'] = { + 'scheme': 'EMAIL_PASSWORD', + 'authdata': { + 'email': credentials['email'], + 'password': credentials['password'] + } } - } - - -def _is_useridtoken_valid(user_id_token): - """Check if user id token is not expired""" - token_data = json.loads(base64.standard_b64decode(user_id_token['tokendata'])) - return token_data['expiration'] > time.time() diff --git a/resources/lib/services/msl/msl_requests.py b/resources/lib/services/msl/msl_requests.py new file mode 100644 index 000000000..267b88bee --- /dev/null +++ b/resources/lib/services/msl/msl_requests.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +""" + Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) + Copyright (C) 2018 Caphm (original implementation module) + Copyright (C) 2020 Stefano Gottardo + MSL requests + + SPDX-License-Identifier: MIT + See LICENSES/MIT.md for more information. +""" +from __future__ import absolute_import, division, unicode_literals + +import base64 +import json +import re +import time +import zlib + +import requests + +import resources.lib.common as common +from resources.lib.globals import g +from resources.lib.services.msl.exceptions import MSLError +from resources.lib.services.msl.msl_request_builder import MSLRequestBuilder +from resources.lib.services.msl.msl_utils import display_error_info, generate_logblobs_params, EVENT_BIND, ENDPOINTS + +try: # Python 2 + from urllib import urlencode +except ImportError: # Python 3 + from urllib.parse import urlencode + + +class MSLRequests(MSLRequestBuilder): + """Provides methods to make MSL requests""" + + def __init__(self, msl_data=None): + super(MSLRequests, self).__init__() + self.session = requests.session() + self._load_msl_data(msl_data) + self.msl_switch_requested = False + + def _load_msl_data(self, msl_data): + try: + self.crypto.load_msl_data(msl_data) + self.crypto.load_crypto_session(msl_data) + + # Add-on just installed, the service starts but there is no esn + if g.get_esn(): + # This is also done here only try to speed up the loading of manifest + self._check_mastertoken_validity() + except MSLError: + raise + except Exception: # pylint: disable=broad-except + import traceback + common.error(traceback.format_exc()) + + @display_error_info + @common.time_execution(immediate=True) + def perform_key_handshake(self, data=None): + """Perform a key handshake and initialize crypto keys""" + # pylint: disable=unused-argument + esn = data or g.get_esn() + if not esn: + common.info('Cannot perform key handshake, missing ESN') + return False + + common.debug('Performing key handshake. ESN: {}', esn) + response = _process_json_response(self._post(ENDPOINTS['manifest'], self.handshake_request(esn))) + header_data = self.decrypt_header_data(response['headerdata'], False) + self.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) + + # Delete all the user id tokens (are correlated to the previous mastertoken) + self.crypto.clear_user_id_tokens() + common.debug('Key handshake successful') + return True + + def _get_owner_user_id_token(self): + """A way to get the user token id of owner profile""" + # In order to get a user id token of another (non-owner) profile you must make a request with SWITCH_PROFILE + # authentication scheme (a custom authentication for netflix), and this request can be directly included + # in the MSL manifest request. + # But in order to execute this switch profile, you need to have the user id token of the main (owner) profile. + # The only way (found to now) to get it immediately, is send a logblob event request, and save the + # user id token obtained in the response. + common.debug('Requesting logblog') + params = {'reqAttempt': 1, + 'reqPriority': 0, + 'reqName': EVENT_BIND} + url = ENDPOINTS['logblobs'] + '?' + urlencode(params).replace('%2F', '/') + response = self.chunked_request(url, + self.build_request_data('/logblob', generate_logblobs_params()), + g.get_esn(), + force_auth_credential=True) + common.debug('Response of logblob request: {}', response) + + def _check_mastertoken_validity(self): + """Return the mastertoken validity and executes a new key handshake when necessary""" + if self.crypto.mastertoken: + time_now = time.time() + renewable = self.crypto.renewal_window < time_now + expired = self.crypto.expiration <= time_now + else: + renewable = False + expired = True + if expired: + if not self.crypto.mastertoken: + debug_msg = 'Stored MSL data not available, a new key handshake will be performed' + else: + debug_msg = 'Stored MSL data is expired, a new key handshake will be performed' + common.debug(debug_msg) + if self.perform_key_handshake(): + msl_data = json.loads(common.load_file('msl_data.json')) + self.crypto.load_msl_data(msl_data) + self.crypto.load_crypto_session(msl_data) + return self._check_mastertoken_validity() + return {'renewable': renewable, 'expired': expired} + + def _check_user_id_token(self, disable_msl_switch, force_auth_credential=False): + """ + Performs user id token checks and return the auth data + checks: uid token validity, get if needed the owner uid token, set when use the switch + + :param: disable_msl_switch: to be used in requests that cannot make the switch + :param: force_auth_credential: force the use of authentication with credentials + :return: auth data that will be used in MSLRequestBuilder _add_auth_info + """ + # Warning: the user id token contains also contains the identity of the netflix profile + # therefore it is necessary to use the right user id token for the request + current_profile_guid = g.LOCAL_DB.get_active_profile_guid() + owner_profile_guid = g.LOCAL_DB.get_guid_owner_profile() + use_switch_profile = False + user_id_token = None + + if not force_auth_credential: + if current_profile_guid == owner_profile_guid: + # It is not necessary to get a token id because by default MSL it is associated to the main profile + # So you do not even need to run the MSL profile switch + user_id_token = self.crypto.get_user_id_token(current_profile_guid) + # user_id_token can return None when the add-on is installed from scratch, in this case will be used + # the authentication with the user credentials + else: + # The request must be executed from a non-owner profile + # Check if the token for the profile exist and valid + user_id_token = self.crypto.get_user_id_token(current_profile_guid) + if not user_id_token and not disable_msl_switch: + # If it is not there, first check if the main profile token exist and valid + use_switch_profile = True + user_id_token = self.crypto.get_user_id_token(owner_profile_guid) + # If it is not there, you must obtain it before making the MSL switch + if not user_id_token: + self._get_owner_user_id_token() + user_id_token = self.crypto.get_user_id_token(owner_profile_guid) + # Mark msl_switch_requested as True in order to make a bind event request + self.msl_switch_requested = True + return {'use_switch_profile': use_switch_profile, 'user_id_token': user_id_token} + + @common.time_execution(immediate=True) + def chunked_request(self, endpoint, request_data, esn, disable_msl_switch=True, force_auth_credential=False): + """Do a POST request and process the chunked response""" + + mt_validity = self._check_mastertoken_validity() + auth_data = self._check_user_id_token(disable_msl_switch, force_auth_credential) + common.debug('Chunked request will be executed with auth data: {}', auth_data) + + chunked_response = self._process_chunked_response( + self._post(endpoint, self.msl_request(request_data, esn, auth_data)), + mt_validity['renewable'] if mt_validity else None, + save_uid_token_to_owner=auth_data['user_id_token'] is None) + return chunked_response['result'] + + def _post(self, endpoint, request_data): + """Execute a post request""" + common.debug('Executing POST request to {}', endpoint) + start = time.clock() + response = self.session.post(endpoint, request_data) + common.debug('Request took {}s', time.clock() - start) + common.debug('Request returned response with status {}', response.status_code) + response.raise_for_status() + return response + + # pylint: disable=unused-argument + @common.time_execution(immediate=True) + def _process_chunked_response(self, response, mt_renewable, save_uid_token_to_owner=False): + """Parse and decrypt an encrypted chunked response. Raise an error + if the response is plaintext json""" + try: + # if the json() does not fail we have an error because + # the expected response is a chunked json response + return _raise_if_error(response.json()) + except ValueError: + # json() failed so parse and decrypt the chunked response + common.debug('Received encrypted chunked response') + response = _parse_chunks(response.text) + # TODO: sending for the renewal request is not yet implemented + # if mt_renewable: + # # Check if mastertoken is renewed + # self.request_builder.crypto.compare_mastertoken(response['header']['mastertoken']) + + header_data = self.decrypt_header_data(response['header'].get('headerdata')) + + if 'useridtoken' in header_data: + # Save the user id token for the future msl requests + profile_guid = g.LOCAL_DB.get_guid_owner_profile() if save_uid_token_to_owner else\ + g.LOCAL_DB.get_active_profile_guid() + self.crypto.save_user_id_token(profile_guid, header_data['useridtoken']) + # if 'keyresponsedata' in header_data: + # common.debug('Found key handshake in response data') + # # Update current mastertoken + # self.request_builder.crypto.parse_key_response(header_data, True) + decrypted_response = _decrypt_chunks(response['payloads'], self.crypto) + return _raise_if_error(decrypted_response) + + +@common.time_execution(immediate=True) +def _process_json_response(response): + """Execute a post request and expect a JSON response""" + try: + return _raise_if_error(response.json()) + except ValueError: + raise MSLError('Expected JSON response, got {}'.format(response.text)) + + +def _raise_if_error(decoded_response): + raise_error = False + # Catch a manifest/chunk error + if any(key in decoded_response for key in ['error', 'errordata']): + raise_error = True + # Catch a license error + if 'result' in decoded_response and isinstance(decoded_response.get('result'), list): + if 'error' in decoded_response['result'][0]: + raise_error = True + if raise_error: + common.error('Full MSL error information:') + common.error(json.dumps(decoded_response)) + raise MSLError(_get_error_details(decoded_response)) + return decoded_response + + +def _get_error_details(decoded_response): + # Catch a chunk error + if 'errordata' in decoded_response: + return json.loads(base64.standard_b64decode(decoded_response['errordata']))['errormsg'] + # Catch a manifest error + if 'error' in decoded_response: + if decoded_response['error'].get('errorDisplayMessage'): + return decoded_response['error']['errorDisplayMessage'] + # Catch a license error + if 'result' in decoded_response and isinstance(decoded_response.get('result'), list): + if 'error' in decoded_response['result'][0]: + if decoded_response['result'][0]['error'].get('errorDisplayMessage'): + return decoded_response['result'][0]['error']['errorDisplayMessage'] + return 'Unhandled error check log.' + + +@common.time_execution(immediate=True) +def _parse_chunks(message): + header = json.loads(message.split('}}')[0] + '}}') + payloads = re.split(',\"signature\":\"[0-9A-Za-z=/+]+\"}', message.split('}}')[1]) + payloads = [x + '}' for x in payloads][:-1] + return {'header': header, 'payloads': payloads} + + +@common.time_execution(immediate=True) +def _decrypt_chunks(chunks, crypto): + decrypted_payload = '' + for chunk in chunks: + payloadchunk = json.loads(chunk) + payload = payloadchunk.get('payload') + decoded_payload = base64.standard_b64decode(payload) + encryption_envelope = json.loads(decoded_payload) + # Decrypt the text + plaintext = crypto.decrypt( + base64.standard_b64decode(encryption_envelope['iv']), + base64.standard_b64decode(encryption_envelope.get('ciphertext'))) + # unpad the plaintext + plaintext = json.loads(plaintext) + data = plaintext.get('data') + + # uncompress data if compressed + if plaintext.get('compressionalgo') == 'GZIP': + decoded_data = base64.standard_b64decode(data) + data = zlib.decompress(decoded_data, 16 + zlib.MAX_WBITS).decode('utf-8') + else: + data = base64.standard_b64decode(data).decode('utf-8') + + decrypted_payload += data + return json.loads(decrypted_payload) diff --git a/resources/lib/services/msl/msl_utils.py b/resources/lib/services/msl/msl_utils.py new file mode 100644 index 000000000..a1c574ebf --- /dev/null +++ b/resources/lib/services/msl/msl_utils.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +""" + Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) + Copyright (C) 2019 Stefano Gottardo - @CastagnaIT (original implementation module) + MSL utils + + SPDX-License-Identifier: MIT + See LICENSES/MIT.md for more information. +""" +from __future__ import absolute_import, division, unicode_literals + +import json +import random +import re +import time +from functools import wraps + +import xbmcgui + +import resources.lib.kodi.ui as ui +from resources.lib import common +from resources.lib.database.db_utils import TABLE_SESSION +from resources.lib.globals import g +from resources.lib.services.msl.exceptions import MSLError + +try: # Python 2 + unicode +except NameError: # Python 3 + unicode = str # pylint: disable=redefined-builtin + +CHROME_BASE_URL = 'https://www.netflix.com/nq/msl_v1/cadmium/' + +ENDPOINTS = { + 'manifest': CHROME_BASE_URL + 'pbo_manifests/%5E1.0.0/router', # "pbo_manifests/^1.0.0/router" + 'license': CHROME_BASE_URL + 'pbo_licenses/%5E1.0.0/router', + 'events': CHROME_BASE_URL + 'pbo_events/%5E1.0.0/router', + 'logblobs': CHROME_BASE_URL + 'pbo_logblobs/%5E1.0.0/router' +} + +EVENT_START = 'start' # events/start : Video starts +EVENT_STOP = 'stop' # events/stop : Video stops +EVENT_KEEP_ALIVE = 'keepAlive' # events/keepAlive : Update progress status +EVENT_ENGAGE = 'engage' # events/engage : After user interaction (before stop, on skip, on pause) +EVENT_BIND = 'bind' # events/bind : ? + +AUDIO_CHANNELS_CONV = {1: '1.0', 2: '2.0', 6: '5.1', 8: '7.1'} + + +def display_error_info(func): + """Decorator that catches errors raise by the decorated function, + displays an error info dialog in the UI and re-raises the error""" + # pylint: disable=missing-docstring + @wraps(func) + def error_catching_wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as exc: + ui.show_error_info(common.get_local_string(30028), unicode(exc), + unknown_error=not(unicode(exc)), + netflix_error=isinstance(exc, MSLError)) + raise + return error_catching_wrapper + + +def is_media_changed(previous_player_state, player_state): + """Check if there are variations in player state to avoids overhead processing""" + if not previous_player_state: + return True + # To now we do not check subtitle, because to the moment it is not implemented + if player_state['currentvideostream'] != previous_player_state['currentvideostream'] or \ + player_state['currentaudiostream'] != previous_player_state['currentaudiostream']: + return True + return False + + +def update_play_times_duration(play_times, player_state): + """Update the playTimes duration values""" + duration = player_state['elapsed_seconds'] * 1000 + play_times['total'] = duration + play_times['audio'][0]['duration'] = duration + play_times['video'][0]['duration'] = duration + + +def build_media_tag(player_state, manifest): + """Build the playTimes and the mediaId data by parsing manifest and the current player streams used""" + common.fix_locale_languages(manifest['audio_tracks']) + duration = player_state['elapsed_seconds'] * 1000 + + audio_downloadable_id, audio_track_id = _find_audio_data(player_state, manifest) + video_downloadable_id, video_track_id = _find_video_data(player_state, manifest) + # Warning 'currentsubtitle' value in player_state on Kodi 18 + # do not have proprieties like isdefault, isforced, isimpaired + # if in the future the implementation will be done it should be available only on Kodi 19 + # then for now we leave the subtitles as disabled + + text_track_id = 'T:1:1;1;NONE;0;1;' + + play_times = { + 'total': duration, + 'audio': [{ + 'downloadableId': audio_downloadable_id, + 'duration': duration + }], + 'video': [{ + 'downloadableId': video_downloadable_id, + 'duration': duration + }], + 'text': [] + } + + # Format example: "A:1:1;2;en;1;|V:2:1;2;;default;1;CE3;0;|T:1:1;1;NONE;0;1;" + media_id = '|'.join([audio_track_id, video_track_id, text_track_id]) + + return play_times, media_id + + +def _find_audio_data(player_state, manifest): + """ + Find the audio downloadable id and the audio track id + """ + language = common.convert_language_iso(player_state['currentaudiostream']['language']) + channels = AUDIO_CHANNELS_CONV[player_state['currentaudiostream']['channels']] + + for audio_track in manifest['audio_tracks']: + if audio_track['language'] == language and audio_track['channels'] == channels: + # Get the stream dict with the highest bitrate + stream = max(audio_track['streams'], key=lambda x: x['bitrate']) + return stream['downloadable_id'], audio_track['new_track_id'] + # Not found? + raise Exception('build_media_tag: unable to find audio data with language: {}, channels: {}' + .format(language, channels)) + + +def _find_video_data(player_state, manifest): + """ + Find the best match for the video downloadable id and the video track id + """ + codec = player_state['currentvideostream']['codec'] + width = player_state['currentvideostream']['width'] + height = player_state['currentvideostream']['height'] + for video_track in manifest['video_tracks']: + for stream in video_track['streams']: + if codec in stream['content_profile'] and width == stream['res_w'] and height == stream['res_h']: + return stream['downloadable_id'], video_track['new_track_id'] + # Not found? + raise Exception('build_media_tag: unable to find video data with codec: {}, width: {}, height: {}' + .format(codec, width, height)) + + +def generate_logblobs_params(): + """Generate the initial log blog""" + # It seems that this log is sent when logging in to a profile the first time + # i think it is the easiest to reproduce, the others contain too much data + screen_size = str(xbmcgui.getScreenWidth()) + 'x' + str(xbmcgui.getScreenHeight()) + timestamp_utc = time.time() + timestamp = int(timestamp_utc * 1000) + client_ver = g.LOCAL_DB.get_value('asset_core', '', table=TABLE_SESSION) + app_id = int(time.time()) * 10000 + random.randint(1, 10001) # Should be used with all log requests + if client_ver: + result = re.search(r'-([0-9\.]+)\.js$', client_ver) + client_ver = result.groups()[0] + + # Here you have to enter only the real data, falsifying the data would cause repercussions in netflix server logs + # therefore since it is possible to exclude data, we avoid entering data that we do not have + blob = { + 'browserua': common.get_user_agent().replace(' ', '#'), + 'browserhref': 'https://www.netflix.com/browse', + # 'initstart': 988, + # 'initdelay': 268, + 'screensize': screen_size, # '1920x1080', + 'screenavailsize': screen_size, # '1920x1040', + 'clientsize': screen_size, # '1920x944', + # 'pt_navigationStart': -1880, + # 'pt_fetchStart': -1874, + # 'pt_secureConnectionStart': -1880, + # 'pt_requestStart': -1853, + # 'pt_domLoading': -638, + # 'm_asl_start': 990, + # 'm_stf_creat': 993, + # 'm_idb_open': 993, + # 'm_idb_succ': 1021, + # 'm_msl_load_no_data': 1059, + # 'm_asl_comp': 1256, + 'type': 'startup', + 'sev': 'info', + 'devmod': 'chrome-cadmium', + 'clver': client_ver, # e.g. '6.0021.220.051' + 'osplatform': g.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION), + 'osver': g.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION), + 'browsername': 'Chrome', + 'browserver': g.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION), + 'appLogSeqNum': 0, + 'uniqueLogId': common.get_random_uuid(), + 'appId': app_id, + 'esn': g.get_esn(), + 'lver': '', + # 'jssid': '15822792997793', # Same value of appId + # 'jsoffms': 1261, + 'clienttime': timestamp, + 'client_utc': timestamp_utc, + 'uiver': g.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION) + } + + blobs_container = { + 'entries': [blob] + } + blobs_dump = json.dumps(blobs_container) + blobs_dump = blobs_dump.replace('"', '\"').replace(' ', '').replace('#', ' ') + return {'logblobs': blobs_dump} diff --git a/resources/lib/services/playback/progress_manager.py b/resources/lib/services/playback/progress_manager.py index fb06f9686..acf03375c 100644 --- a/resources/lib/services/playback/progress_manager.py +++ b/resources/lib/services/playback/progress_manager.py @@ -12,8 +12,7 @@ from xbmcgui import Window import resources.lib.common as common -from resources.lib.globals import g -from resources.lib.services.msl.events_handler import EVENT_STOP, EVENT_KEEP_ALIVE, EVENT_START, EVENT_ENGAGE +from resources.lib.services.msl.msl_utils import EVENT_START, EVENT_ENGAGE, EVENT_STOP, EVENT_KEEP_ALIVE from .action_manager import PlaybackActionManager @@ -32,15 +31,6 @@ def __init__(self): # pylint: disable=super-on-old-class self.window_cls = Window(10000) # Kodi home window def _initialize(self, data): - if not g.LOCAL_DB.get_profile_config('isAccountOwner', False): - # Currently due to a unknown problem, it is not possible to communicate MSL data to the right selected - # profile other than the owner profile - self.enabled = False - return - videoid = common.VideoId.from_dict(data['videoid']) - if videoid.mediatype not in [common.VideoId.MOVIE, common.VideoId.EPISODE]: - self.enabled = False - return if not data['event_data']: common.warn('ProgressManager: disabled due to no event data') self.enabled = False diff --git a/resources/lib/services/playback/stream_continuity.py b/resources/lib/services/playback/stream_continuity.py index 9f06567ea..a9fb9330c 100644 --- a/resources/lib/services/playback/stream_continuity.py +++ b/resources/lib/services/playback/stream_continuity.py @@ -240,7 +240,8 @@ def _show_only_forced_subtitle(self): # --- ONLY FOR KODI VERSION 18 --- # NOTE: With Kodi 18 it is not possible to read the properties of the streams # so the only possible way is to read the data from the manifest file - cache_identifier = g.get_esn() + '_' + self.current_videoid.value + cache_identifier = (g.LOCAL_DB.get_active_profile_guid() + '_' + + g.get_esn() + '_' + self.current_videoid.value) manifest_data = g.CACHE.get(CACHE_MANIFESTS, cache_identifier, False) common.fix_locale_languages(manifest_data['timedtexttracks']) if not any(text_track.get('isForcedNarrative', False) is True and diff --git a/resources/lib/upgrade_controller.py b/resources/lib/upgrade_controller.py index a4e2b81e6..0a8090564 100644 --- a/resources/lib/upgrade_controller.py +++ b/resources/lib/upgrade_controller.py @@ -86,7 +86,9 @@ def _perform_shared_db_changes(current_version, upgrade_to_version): # Init fix from resources.lib.common import is_minimum_version service_previous_ver = g.LOCAL_DB.get_value('service_previous_version', None) - if current_version is None and not is_minimum_version(service_previous_ver, '0.17.0'): + if service_previous_ver is not None and\ + current_version is None and\ + not is_minimum_version(service_previous_ver, '0.17.0'): current_version = '0.1' # End fix diff --git a/test/xbmcgui.py b/test/xbmcgui.py index 16fe70179..fc2a0376d 100644 --- a/test/xbmcgui.py +++ b/test/xbmcgui.py @@ -16,6 +16,16 @@ ALPHANUM_HIDE_INPUT = 1 +def getScreenHeight(): + """A stub implementation of the xbmcgui getScreenHeight() function""" + return 1080 + + +def getScreenWidth(): + """A stub implementation of the xbmcgui getScreenWidth() function""" + return 1920 + + class Control: """A reimplementation of the xbmcgui Control class"""