From 4bf20b96bb25850d083ab07b629acf7ece94da9a Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 10:37:09 +0100 Subject: [PATCH 01/11] Do not change pylint max line length --- .editorconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/.editorconfig b/.editorconfig index dca80abd4..ab2b975e8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,8 +14,6 @@ charset = utf-8 trim_trailing_whitespace = true # A file must end with an empty line - this is good for version control systems insert_final_newline = true -# A line should not have more than this amount of chars (not supported by all plugins) -max_line_length = 100 [*.{py,md,txt}] indent_size = 4 From e77aca7771bb0354d97bf04194ef8b241039c650 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 11:13:22 +0100 Subject: [PATCH 02/11] Reworked msl classes *there are some revert back to allow you to understand the changes in the follow commit --- resources/lib/services/msl/android_crypto.py | 29 ++-- resources/lib/services/msl/base_crypto.py | 29 ++-- resources/lib/services/msl/default_crypto.py | 19 ++- resources/lib/services/msl/events_handler.py | 23 ++- resources/lib/services/msl/msl_handler.py | 68 ++++---- ...uest_builder.py => msl_request_builder.py} | 36 ++-- .../{msl_handler_base.py => msl_requests.py} | 154 +++++++----------- .../{event_tag_builder.py => msl_utils.py} | 42 ++++- .../lib/services/playback/progress_manager.py | 2 +- 9 files changed, 203 insertions(+), 199 deletions(-) rename resources/lib/services/msl/{request_builder.py => msl_request_builder.py} (86%) rename resources/lib/services/msl/{msl_handler_base.py => msl_requests.py} (60%) rename resources/lib/services/msl/{event_tag_builder.py => msl_utils.py} (71%) diff --git a/resources/lib/services/msl/android_crypto.py b/resources/lib/services/msl/android_crypto.py index 0c2aabb41..bbe6a1860 100644 --- a/resources/lib/services/msl/android_crypto.py +++ b/resources/lib/services/msl/android_crypto.py @@ -23,28 +23,21 @@ class AndroidMSLCrypto(MSLBaseCrypto): """Crypto handler for Android platforms""" - def __init__(self, msl_data=None): # pylint: disable=super-on-old-class - # pylint: disable=broad-except + def __init__(self): + super(AndroidMSLCrypto, self).__init__() + self.crypto_session = None + self.keyset_id = None + self.key_id = None + self.hmac_key_id = None try: self.crypto_session = xbmcdrm.CryptoSession( 'edef8ba9-79d6-4ace-a3c8-27dcd51d21ed', 'AES/CBC/NoPadding', 'HmacSHA256') common.debug('Widevine CryptoSession successful constructed') - except Exception: + except Exception: # pylint: disable=broad-except import traceback common.error(traceback.format_exc()) raise MSLError('Failed to construct Widevine CryptoSession') - try: - super(AndroidMSLCrypto, self).__init__(msl_data) - self.keyset_id = base64.standard_b64decode(msl_data['key_set_id']) - self.key_id = base64.standard_b64decode(msl_data['key_id']) - self.hmac_key_id = base64.standard_b64decode(msl_data['hmac_key_id']) - self.crypto_session.RestoreKeys(self.keyset_id) - except Exception: - self.keyset_id = None - self.key_id = None - self.hmac_key_id = None - drm_info = { 'version': self.crypto_session.GetPropertyString('version'), 'system_id': self.crypto_session.GetPropertyString('systemId'), @@ -75,6 +68,14 @@ def __init__(self, msl_data=None): # pylint: disable=super-on-old-class common.debug('Widevine CryptoSession max hdcp level supported', drm_info['hdcp_level_max']) common.debug('Widevine CryptoSession algorithms: {}', self.crypto_session.GetPropertyString('algorithms')) + def load_crypto_session(self, msl_data=None): + if not msl_data: + return + self.keyset_id = base64.standard_b64decode(msl_data['key_set_id']) + self.key_id = base64.standard_b64decode(msl_data['key_id']) + self.hmac_key_id = base64.standard_b64decode(msl_data['hmac_key_id']) + self.crypto_session.RestoreKeys(self.keyset_id) + def __del__(self): self.crypto_session = None diff --git a/resources/lib/services/msl/base_crypto.py b/resources/lib/services/msl/base_crypto.py index 29511fdb7..c1a025336 100644 --- a/resources/lib/services/msl/base_crypto.py +++ b/resources/lib/services/msl/base_crypto.py @@ -18,18 +18,25 @@ class MSLBaseCrypto(object): """Common base class for MSL crypto operations. Handles mastertoken and sequence number""" - # pylint: disable=too-few-public-methods - def __init__(self, msl_data=None): + + def __init__(self): + self._msl_data = None + self.mastertoken = None + self.serial_number = None + self.sequence_number = None + self.renewal_window = None + self.expiration = None + + def load_msl_data(self, msl_data=None): + self._msl_data = msl_data if msl_data else {} if msl_data: - self._set_mastertoken(msl_data['tokens']['mastertoken']) - else: - self.mastertoken = None + self.set_mastertoken(msl_data['tokens']['mastertoken']) def compare_mastertoken(self, mastertoken): """Check if the new mastertoken is different from current due to renew""" if not self._mastertoken_is_newer_that(mastertoken): common.debug('MSL mastertoken is changed due to renew') - self._set_mastertoken(mastertoken) + self.set_mastertoken(mastertoken) self._save_msl_data() def _mastertoken_is_newer_that(self, mastertoken): @@ -48,12 +55,12 @@ def _mastertoken_is_newer_that(self, mastertoken): def parse_key_response(self, headerdata, save_to_disk): """Parse a key response and update crypto keys""" - self._set_mastertoken(headerdata['keyresponsedata']['mastertoken']) + self.set_mastertoken(headerdata['keyresponsedata']['mastertoken']) self._init_keys(headerdata['keyresponsedata']) if save_to_disk: self._save_msl_data() - def _set_mastertoken(self, mastertoken): + def set_mastertoken(self, mastertoken): """Set the mastertoken and check it for validity""" tokendata = json.loads( base64.standard_b64decode(mastertoken['tokendata'].encode('utf-8')).decode('utf-8')) @@ -65,9 +72,9 @@ def _set_mastertoken(self, mastertoken): def _save_msl_data(self): """Save crypto keys and mastertoken to disk""" - msl_data = {'tokens': {'mastertoken': self.mastertoken}} - msl_data.update(self._export_keys()) - common.save_file('msl_data.json', json.dumps(msl_data).encode('utf-8')) + self._msl_data['tokens'] = {'mastertoken': self.mastertoken} + self._msl_data.update(self._export_keys()) + common.save_file('msl_data.json', json.dumps(self._msl_data).encode('utf-8')) common.debug('Successfully saved MSL data to disk') def _init_keys(self, key_response_data): diff --git a/resources/lib/services/msl/default_crypto.py b/resources/lib/services/msl/default_crypto.py index fee59996c..3feff711d 100644 --- a/resources/lib/services/msl/default_crypto.py +++ b/resources/lib/services/msl/default_crypto.py @@ -10,8 +10,9 @@ """ from __future__ import absolute_import, division, unicode_literals -import json import base64 +import json + try: # Python 3 from Crypto.Random import get_random_bytes from Crypto.Hash import HMAC, SHA256 @@ -30,23 +31,29 @@ import resources.lib.common as common from .base_crypto import MSLBaseCrypto +from .exceptions import MSLError class DefaultMSLCrypto(MSLBaseCrypto): """Crypto Handler for non-Android platforms""" - def __init__(self, msl_data=None): # pylint: disable=super-on-old-class - # pylint: disable=broad-except + + def __init__(self): + super(DefaultMSLCrypto, self).__init__() + self.rsa_key = None + self.encryption_key = None + self.sign_key = None + + def load_crypto_session(self, msl_data=None): try: - super(DefaultMSLCrypto, self).__init__(msl_data) self.encryption_key = base64.standard_b64decode( msl_data['encryption_key']) self.sign_key = base64.standard_b64decode( msl_data['sign_key']) if not self.encryption_key or not self.sign_key: - raise ValueError('Missing encryption_key or sign_key') + raise MSLError('Missing encryption_key or sign_key') self.rsa_key = RSA.importKey( base64.standard_b64decode(msl_data['rsa_key'])) - except Exception: + except Exception: # pylint: disable=broad-except common.debug('Generating new RSA keys') self.rsa_key = RSA.generate(2048) self.encryption_key = None diff --git a/resources/lib/services/msl/events_handler.py b/resources/lib/services/msl/events_handler.py index f2444e267..a3c735ef0 100644 --- a/resources/lib/services/msl/events_handler.py +++ b/resources/lib/services/msl/events_handler.py @@ -25,20 +25,14 @@ from resources.lib import common from resources.lib.database.db_utils import TABLE_SESSION from resources.lib.globals import g -from resources.lib.services.msl import event_tag_builder -from resources.lib.services.msl.msl_handler_base import build_request_data, ENDPOINTS +from resources.lib.services.msl import msl_utils +from resources.lib.services.msl.msl_utils import EVENT_START, EVENT_STOP, EVENT_ENGAGE, ENDPOINTS try: import Queue as queue except ImportError: # Python 3 import queue -EVENT_START = 'start' # events/start : Video starts -EVENT_STOP = 'stop' # events/stop : Video stops -EVENT_KEEP_ALIVE = 'keepAlive' # events/keepAlive : Update progress status -EVENT_ENGAGE = 'engage' # events/engage : After user interaction (before stop, on skip, on pause) -EVENT_BIND = 'bind' # events/bind : ? - class Event(object): """Object representing an event request to be processed""" @@ -160,7 +154,12 @@ def add_event_to_queue(self, event_type, event_data, player_state): event_type, previous_data.get('xid')) return - event_data = build_request_data(url, self._build_event_params(event_type, event_data, player_state, manifest)) + from resources.lib.services.msl.msl_request_builder import MSLRequestBuilder + event_data = MSLRequestBuilder.build_request_data(url, + self._build_event_params(event_type, + event_data, + player_state, + manifest)) try: self.queue_events.put_nowait(Event(event_data)) except queue.Full: @@ -192,11 +191,11 @@ def _build_event_params(self, event_type, event_data, player_state, manifest): # else: # list_id = g.LOCAL_DB.get_value('last_menu_id', 'unknown') - if event_tag_builder.is_media_changed(previous_player_state, player_state): - play_times, media_id = event_tag_builder.build_media_tag(player_state, manifest) + if msl_utils.is_media_changed(previous_player_state, player_state): + play_times, media_id = msl_utils.build_media_tag(player_state, manifest) else: play_times = previous_data['playTimes'] - event_tag_builder.update_play_times_duration(play_times, player_state) + msl_utils.update_play_times_duration(play_times, player_state) media_id = previous_data['mediaId'] params = { diff --git a/resources/lib/services/msl/msl_handler.py b/resources/lib/services/msl/msl_handler.py index ec68eef53..a27d04d04 100644 --- a/resources/lib/services/msl/msl_handler.py +++ b/resources/lib/services/msl/msl_handler.py @@ -2,7 +2,7 @@ """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) Copyright (C) 2017 Trummerjo (original implementation module) - Proxy service to convert manifest and provide license data + Proxy service to convert manifest, provide license data and handle events SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. @@ -12,7 +12,6 @@ import json import time -import requests import xbmcaddon import resources.lib.cache as cache @@ -21,9 +20,9 @@ from resources.lib.globals import g from .converter import convert_to_dash from .events_handler import EventsHandler -from .msl_handler_base import MSLHandlerBase, ENDPOINTS, display_error_info, build_request_data +from .msl_requests import MSLRequests +from .msl_utils import ENDPOINTS, display_error_info from .profiles import enabled_profiles -from .request_builder import MSLRequestBuilder try: # Python 2 unicode @@ -31,39 +30,31 @@ unicode = str # pylint: disable=redefined-builtin -class MSLHandler(MSLHandlerBase): - """Handles session management and crypto for license and manifest - requests""" +class MSLHandler(object): + """Handles session management and crypto for license, manifest and event requests""" last_license_session_id = '' last_license_url = '' last_license_release_url = '' last_drm_context = '' last_playback_context = '' - session = requests.session() def __init__(self): super(MSLHandler, self).__init__() - # pylint: disable=broad-except self.request_builder = None try: msl_data = json.loads(common.load_file('msl_data.json')) common.info('Loaded MSL data from disk') - except Exception: + except Exception: # pylint: disable=broad-except msl_data = None - try: - self.request_builder = MSLRequestBuilder(msl_data) - # Addon just installed, the service starts but there is no esn - if g.get_esn(): - self.check_mastertoken_validity() - - events_handler = EventsHandler(self.chunked_request) - events_handler.start() - except Exception: - import traceback - common.error(traceback.format_exc()) + + self.request_builder = MSLRequests(msl_data) + + events_handler = EventsHandler(self.request_builder.chunked_request) + events_handler.start() + common.register_slot( signal=common.Signals.ESN_CHANGED, - callback=self.perform_key_handshake) + callback=self.request_builder.perform_key_handshake) common.register_slot( signal=common.Signals.RELEASE_LICENSE, callback=self.release_license) @@ -93,13 +84,13 @@ def get_edge_manifest(self, viewable_id, chrome_manifest): common.debug('Loading EDGE manifest') esn = g.get_edge_esn() common.debug('Switching MSL data to EDGE') - self.perform_key_handshake(esn) + self.request_builder.perform_key_handshake(esn) manifest = self._load_manifest(viewable_id, esn) manifest['playbackContextId'] = chrome_manifest['playbackContextId'] manifest['drmContextId'] = chrome_manifest['drmContextId'] common.debug('Successfully loaded EDGE manifest') common.debug('Resetting MSL data to Chrome') - self.perform_key_handshake() + self.request_builder.perform_key_handshake() return manifest @common.time_execution(immediate=True) @@ -169,12 +160,9 @@ def _load_manifest(self, viewable_id, esn): 'preferAssistiveAudio': False } - # Get and check mastertoken validity - mt_validity = self.check_mastertoken_validity() - manifest = self.chunked_request(ENDPOINTS['manifest'], - build_request_data('/manifest', params), - esn, - mt_validity) + manifest = self.request_builder.chunked_request(ENDPOINTS['manifest'], + self.request_builder.build_request_data('/manifest', params), + esn) if common.is_debug_verbose(): # Save the manifest to disk as reference common.save_file('manifest.json', json.dumps(manifest).encode('utf-8')) @@ -198,22 +186,22 @@ def get_license(self, challenge, sid): timestamp = int(time.time() * 10000) xid = str(timestamp + 1610) - params = [{ 'sessionId': sid, 'clientTime': int(timestamp / 10000), 'challengeBase64': challenge, 'xid': xid }] - - response = self.chunked_request(ENDPOINTS['license'], - build_request_data(self.last_license_url, params, 'sessionId'), - g.get_esn()) - + response = self.request_builder.chunked_request(ENDPOINTS['license'], + self.request_builder.build_request_data(self.last_license_url, + params, + 'sessionId'), + g.get_esn()) # This xid must be used for any future request, until playback stops g.LOCAL_DB.set_value('xid', xid, TABLE_SESSION) self.last_license_session_id = sid self.last_license_release_url = response[0]['links']['releaseLicense']['href'] + return response[0]['licenseResponseBase64'] @display_error_info @@ -222,7 +210,7 @@ def release_license(self, data=None): # pylint: disable=unused-argument """ Release the server license """ - common.debug('Releasing license') + common.debug('Requesting releasing license') params = [{ 'url': self.last_license_release_url, @@ -233,9 +221,9 @@ def release_license(self, data=None): # pylint: disable=unused-argument 'echo': 'sessionId' }] - response = self.chunked_request(ENDPOINTS['license'], - build_request_data('/bundle', params), - g.get_esn()) + response = self.request_builder.chunked_request(ENDPOINTS['license'], + self.request_builder.build_request_data('/bundle', params), + g.get_esn()) common.debug('License release response: {}', response) @common.time_execution(immediate=True) diff --git a/resources/lib/services/msl/request_builder.py b/resources/lib/services/msl/msl_request_builder.py similarity index 86% rename from resources/lib/services/msl/request_builder.py rename to resources/lib/services/msl/msl_request_builder.py index 091c496fa..8f9764de3 100644 --- a/resources/lib/services/msl/request_builder.py +++ b/resources/lib/services/msl/msl_request_builder.py @@ -35,11 +35,24 @@ class MSLRequestBuilder(object): """Provides mechanisms to create MSL requests""" - def __init__(self, msl_data=None): + def __init__(self): self.current_message_id = None - self.user_id_token = None self.rndm = random.SystemRandom() - self.crypto = MSLCrypto(msl_data) + self.crypto = MSLCrypto() + + @staticmethod + def build_request_data(url, params=None, echo=''): + """Create a standard request data""" + timestamp = int(time.time() * 10000) + request_data = { + 'version': 2, + 'url': url, + 'id': timestamp, + 'languages': [g.LOCAL_DB.get_profile_config('language')], + 'params': params, + 'echo': echo + } + return request_data @common.time_execution(immediate=True) def msl_request(self, data, esn): @@ -91,7 +104,7 @@ def _headerdata(self, esn=None, compression=None, is_handshake=False): header_data['keyrequestdata'] = self.crypto.key_request_data() else: header_data['sender'] = esn - _add_auth_info(header_data, self.user_id_token) + self._add_auth_info(header_data) return json.dumps(header_data) @@ -121,13 +134,8 @@ def decrypt_header_data(self, data, enveloped=True): return json.loads(self.crypto.decrypt(init_vector, cipher_text)) return header_data - -def _add_auth_info(header_data, user_id_token): - """User authentication identifies the application user associated with a message""" - if user_id_token and _is_useridtoken_valid(user_id_token): - # Authentication with user ID token containing the user identity - header_data['useridtoken'] = user_id_token - else: + def _add_auth_info(self, header_data): + """User authentication identifies the application user associated with a message""" # Authentication with the user credentials credentials = common.get_credentials() header_data['userauthdata'] = { @@ -137,9 +145,3 @@ def _add_auth_info(header_data, user_id_token): 'password': credentials['password'] } } - - -def _is_useridtoken_valid(user_id_token): - """Check if user id token is not expired""" - token_data = json.loads(base64.standard_b64decode(user_id_token['tokendata'])) - return token_data['expiration'] > time.time() diff --git a/resources/lib/services/msl/msl_handler_base.py b/resources/lib/services/msl/msl_requests.py similarity index 60% rename from resources/lib/services/msl/msl_handler_base.py rename to resources/lib/services/msl/msl_requests.py index 2570a5f80..424e8ff98 100644 --- a/resources/lib/services/msl/msl_handler_base.py +++ b/resources/lib/services/msl/msl_requests.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) - Copyright (C) 2017 Trummerjo (original implementation module) - Proxy service to convert manifest and provide license data + Copyright (C) 2018 Caphm (original implementation module) + MSL request building SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. @@ -14,76 +14,36 @@ import re import time import zlib -from functools import wraps import requests import resources.lib.common as common -import resources.lib.kodi.ui as ui from resources.lib.globals import g from resources.lib.services.msl.exceptions import MSLError -from resources.lib.services.msl.request_builder import MSLRequestBuilder +from resources.lib.services.msl.msl_request_builder import MSLRequestBuilder +from resources.lib.services.msl.msl_utils import display_error_info, ENDPOINTS -try: # Python 2 - unicode -except NameError: # Python 3 - unicode = str # pylint: disable=redefined-builtin +class MSLRequests(MSLRequestBuilder): -CHROME_BASE_URL = 'https://www.netflix.com/nq/msl_v1/cadmium/' -ENDPOINTS = { - 'manifest': CHROME_BASE_URL + 'pbo_manifests/%5E1.0.0/router', # "pbo_manifests/^1.0.0/router" - 'license': CHROME_BASE_URL + 'pbo_licenses/%5E1.0.0/router', - 'events': CHROME_BASE_URL + 'pbo_events/%5E1.0.0/router' -} + def __init__(self, msl_data=None): + super(MSLRequests, self).__init__() + self.session = requests.session() + self._load_msl_data(msl_data) - -def display_error_info(func): - """Decorator that catches errors raise by the decorated function, - displays an error info dialog in the UI and reraises the error""" - # pylint: disable=missing-docstring - @wraps(func) - def error_catching_wrapper(*args, **kwargs): + def _load_msl_data(self, msl_data): try: - return func(*args, **kwargs) - except Exception as exc: - ui.show_error_info(common.get_local_string(30028), unicode(exc), - unknown_error=not(unicode(exc)), - netflix_error=isinstance(exc, MSLError)) - raise - return error_catching_wrapper - - -class MSLHandlerBase(object): - """Handles session management and crypto for license, manifest and event requests""" - last_license_url = '' - last_drm_context = '' - last_playback_context = '' - session = requests.session() + self.crypto.load_msl_data(msl_data) + self.crypto.load_crypto_session(msl_data) - def __init__(self): - self.request_builder = None - - def check_mastertoken_validity(self): - """Return the mastertoken validity and executes a new key handshake when necessary""" - if self.request_builder.crypto.mastertoken: - time_now = time.time() - renewable = self.request_builder.crypto.renewal_window < time_now - expired = self.request_builder.crypto.expiration <= time_now - else: - renewable = False - expired = True - if expired: - if not self.request_builder.crypto.mastertoken: - common.debug('Stored MSL data not available, a new key handshake will be performed') - self.request_builder = MSLRequestBuilder() - else: - common.debug('Stored MSL data is expired, a new key handshake will be performed') - if self.perform_key_handshake(): - self.request_builder = MSLRequestBuilder(json.loads( - common.load_file('msl_data.json'))) - return self.check_mastertoken_validity() - return {'renewable': renewable, 'expired': expired} + # Add-on just installed, the service starts but there is no esn + if g.get_esn(): + self._check_mastertoken_validity() + except MSLError: + raise + except Exception: # pylint: disable=broad-except + import traceback + common.error(traceback.format_exc()) @display_error_info @common.time_execution(immediate=True) @@ -97,21 +57,45 @@ def perform_key_handshake(self, data=None): common.debug('Performing key handshake. ESN: {}', esn) - response = _process_json_response( - self._post(ENDPOINTS['manifest'], - self.request_builder.handshake_request(esn))) - header_data = self.request_builder.decrypt_header_data(response['headerdata'], False) - self.request_builder.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) - # Reset the user id token - self.request_builder.user_id_token = None + response = _process_json_response(self._post(ENDPOINTS['manifest'], self.handshake_request(esn))) + header_data = self.decrypt_header_data(response['headerdata'], False) + self.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) + + # Delete all the user id tokens (are correlated to the previous mastertoken) + # self.crypto.clear_user_id_tokens() common.debug('Key handshake successful') return True + def _check_mastertoken_validity(self): + """Return the mastertoken validity and executes a new key handshake when necessary""" + if self.crypto.mastertoken: + time_now = time.time() + renewable = self.crypto.renewal_window < time_now + expired = self.crypto.expiration <= time_now + else: + renewable = False + expired = True + if expired: + if not self.crypto.mastertoken: + debug_msg = 'Stored MSL data not available, a new key handshake will be performed' + else: + debug_msg = 'Stored MSL data is expired, a new key handshake will be performed' + common.debug(debug_msg) + if self.perform_key_handshake(): + msl_data = json.loads(common.load_file('msl_data.json')) + self.crypto.load_msl_data(msl_data) + self.crypto.load_crypto_session(msl_data) + return self._check_mastertoken_validity() + return {'renewable': renewable, 'expired': expired} + @common.time_execution(immediate=True) - def chunked_request(self, endpoint, request_data, esn, mt_validity=None): + def chunked_request(self, endpoint, request_data, esn): """Do a POST request and process the chunked response""" + + mt_validity = self._check_mastertoken_validity() + chunked_response = self._process_chunked_response( - self._post(endpoint, self.request_builder.msl_request(request_data, esn)), + self._post(endpoint, self.msl_request(request_data, esn)), mt_validity['renewable'] if mt_validity else None) return chunked_response['result'] @@ -144,37 +128,16 @@ def _process_chunked_response(self, response, mt_renewable): # # Check if mastertoken is renewed # self.request_builder.crypto.compare_mastertoken(response['header']['mastertoken']) - header_data = self.request_builder.decrypt_header_data( - response['header'].get('headerdata')) + # header_data = self.decrypt_header_data(response['header'].get('headerdata')) - if 'useridtoken' in header_data: - # After the first call, it is possible get the 'user id token' that contains the - # user identity to use instead of 'User Authentication Data' with user credentials - self.request_builder.user_id_token = header_data['useridtoken'] # if 'keyresponsedata' in header_data: # common.debug('Found key handshake in response data') # # Update current mastertoken # self.request_builder.crypto.parse_key_response(header_data, True) - decrypted_response = _decrypt_chunks(response['payloads'], self.request_builder.crypto) + decrypted_response = _decrypt_chunks(response['payloads'], self.crypto) return _raise_if_error(decrypted_response) -def build_request_data(url, params=None, echo=''): - """Create a standard request data""" - if not params: - raise Exception('Cannot build the message without parameters') - timestamp = int(time.time() * 10000) - request_data = { - 'version': 2, - 'url': url, - 'id': timestamp, - 'languages': [g.LOCAL_DB.get_profile_config('language')], - 'params': params, - 'echo': echo - } - return request_data - - @common.time_execution(immediate=True) def _process_json_response(response): """Execute a post request and expect a JSON response""" @@ -203,9 +166,7 @@ def _raise_if_error(decoded_response): def _get_error_details(decoded_response): # Catch a chunk error if 'errordata' in decoded_response: - return json.loads( - base64.standard_b64decode( - decoded_response['errordata']))['errormsg'] + return json.loads(base64.standard_b64decode(decoded_response['errordata']))['errormsg'] # Catch a manifest error if 'error' in decoded_response: if decoded_response['error'].get('errorDisplayMessage'): @@ -221,8 +182,7 @@ def _get_error_details(decoded_response): @common.time_execution(immediate=True) def _parse_chunks(message): header = json.loads(message.split('}}')[0] + '}}') - payloads = re.split(',\"signature\":\"[0-9A-Za-z=/+]+\"}', - message.split('}}')[1]) + payloads = re.split(',\"signature\":\"[0-9A-Za-z=/+]+\"}', message.split('}}')[1]) payloads = [x + '}' for x in payloads][:-1] return {'header': header, 'payloads': payloads} diff --git a/resources/lib/services/msl/event_tag_builder.py b/resources/lib/services/msl/msl_utils.py similarity index 71% rename from resources/lib/services/msl/event_tag_builder.py rename to resources/lib/services/msl/msl_utils.py index 22ba6154c..621a4452c 100644 --- a/resources/lib/services/msl/event_tag_builder.py +++ b/resources/lib/services/msl/msl_utils.py @@ -2,18 +2,58 @@ """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) Copyright (C) 2019 Stefano Gottardo - @CastagnaIT (original implementation module) - Build event tags values + Msl utils SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. """ from __future__ import absolute_import, division, unicode_literals +from functools import wraps + +import resources.lib.kodi.ui as ui from resources.lib import common +from resources.lib.services.msl.exceptions import MSLError + +try: # Python 2 + unicode +except NameError: # Python 3 + unicode = str # pylint: disable=redefined-builtin + +CHROME_BASE_URL = 'https://www.netflix.com/nq/msl_v1/cadmium/' + +ENDPOINTS = { + 'manifest': CHROME_BASE_URL + 'pbo_manifests/%5E1.0.0/router', # "pbo_manifests/^1.0.0/router" + 'license': CHROME_BASE_URL + 'pbo_licenses/%5E1.0.0/router', + 'events': CHROME_BASE_URL + 'pbo_events/%5E1.0.0/router', + 'logblobs': CHROME_BASE_URL + 'pbo_logblobs/%5E1.0.0/router' +} + +EVENT_START = 'start' # events/start : Video starts +EVENT_STOP = 'stop' # events/stop : Video stops +EVENT_KEEP_ALIVE = 'keepAlive' # events/keepAlive : Update progress status +EVENT_ENGAGE = 'engage' # events/engage : After user interaction (before stop, on skip, on pause) +EVENT_BIND = 'bind' # events/bind : ? AUDIO_CHANNELS_CONV = {1: '1.0', 2: '2.0', 6: '5.1', 8: '7.1'} +def display_error_info(func): + """Decorator that catches errors raise by the decorated function, + displays an error info dialog in the UI and re-raises the error""" + # pylint: disable=missing-docstring + @wraps(func) + def error_catching_wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as exc: + ui.show_error_info(common.get_local_string(30028), unicode(exc), + unknown_error=not(unicode(exc)), + netflix_error=isinstance(exc, MSLError)) + raise + return error_catching_wrapper + + def is_media_changed(previous_player_state, player_state): """Check if there are variations in player state to avoids overhead processing""" if not previous_player_state: diff --git a/resources/lib/services/playback/progress_manager.py b/resources/lib/services/playback/progress_manager.py index fb06f9686..ee4cc5a7a 100644 --- a/resources/lib/services/playback/progress_manager.py +++ b/resources/lib/services/playback/progress_manager.py @@ -13,7 +13,7 @@ import resources.lib.common as common from resources.lib.globals import g -from resources.lib.services.msl.events_handler import EVENT_STOP, EVENT_KEEP_ALIVE, EVENT_START, EVENT_ENGAGE +from resources.lib.services.msl.msl_utils import EVENT_START, EVENT_ENGAGE, EVENT_STOP, EVENT_KEEP_ALIVE from .action_manager import PlaybackActionManager From 4fb863dc16bfc90091d427e823dba9ea178a4a44 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 11:38:28 +0100 Subject: [PATCH 03/11] Implemented the MSL switch profile --- resources/lib/services/msl/base_crypto.py | 30 +++++++ resources/lib/services/msl/msl_handler.py | 20 ++++- .../lib/services/msl/msl_request_builder.py | 51 +++++++---- resources/lib/services/msl/msl_requests.py | 85 +++++++++++++++++-- resources/lib/services/msl/msl_utils.py | 70 +++++++++++++++ 5 files changed, 232 insertions(+), 24 deletions(-) diff --git a/resources/lib/services/msl/base_crypto.py b/resources/lib/services/msl/base_crypto.py index c1a025336..d24427483 100644 --- a/resources/lib/services/msl/base_crypto.py +++ b/resources/lib/services/msl/base_crypto.py @@ -11,6 +11,7 @@ import json import base64 +import time import resources.lib.common as common @@ -84,3 +85,32 @@ def _init_keys(self, key_response_data): def _export_keys(self): """Export crypto keys to a dict""" raise NotImplementedError + + def get_user_id_token(self, profile_guid): + """Get a valid the user id token associated to a profile guid""" + if 'user_id_tokens' in self._msl_data: + user_id_token = self._msl_data['user_id_tokens'].get(profile_guid) + if user_id_token and self.is_user_id_token_valid(user_id_token): + return user_id_token + return None + + def save_user_id_token(self, profile_guid, user_token_id): + """Save or update a user id token associated to a profile guid""" + if 'user_id_tokens' not in self._msl_data: + self._msl_data['user_id_tokens'] = { + profile_guid: user_token_id + } + else: + self._msl_data['user_id_tokens'][profile_guid] = user_token_id + self._save_msl_data() + + def clear_user_id_tokens(self): + """Clear all user id tokens""" + self._msl_data.pop('user_id_tokens', None) + self._save_msl_data() + + def is_user_id_token_valid(self, user_id_token): + """Check if user id token is not expired""" + token_data = json.loads(base64.standard_b64decode(user_id_token['tokendata'])) + # Subtract 5min as a safety measure + return (token_data['expiration'] - 300) > time.time() diff --git a/resources/lib/services/msl/msl_handler.py b/resources/lib/services/msl/msl_handler.py index a27d04d04..ce76bcec4 100644 --- a/resources/lib/services/msl/msl_handler.py +++ b/resources/lib/services/msl/msl_handler.py @@ -162,7 +162,8 @@ def _load_manifest(self, viewable_id, esn): manifest = self.request_builder.chunked_request(ENDPOINTS['manifest'], self.request_builder.build_request_data('/manifest', params), - esn) + esn, + disable_msl_switch=False) if common.is_debug_verbose(): # Save the manifest to disk as reference common.save_file('manifest.json', json.dumps(manifest).encode('utf-8')) @@ -202,8 +203,25 @@ def get_license(self, challenge, sid): self.last_license_session_id = sid self.last_license_release_url = response[0]['links']['releaseLicense']['href'] + if self.request_builder.msl_switch_requested: + self.request_builder.msl_switch_requested = False + self.bind_events() return response[0]['licenseResponseBase64'] + def bind_events(self): + """ + Bind events + """ + # I don't know the real purpose of its use, it seems to be requested after the license and before starting + # playback, and only the first time after a switch, + # in the response you can also understand if the msl switch has worked + common.debug('Requesting bind events') + response = self.request_builder.chunked_request(ENDPOINTS['events'], + self.request_builder.build_request_data('/bind', {}), + g.get_esn(), + disable_msl_switch=False) + common.debug('Bind events response: {}', response) + @display_error_info @common.time_execution(immediate=True) def release_license(self, data=None): # pylint: disable=unused-argument diff --git a/resources/lib/services/msl/msl_request_builder.py b/resources/lib/services/msl/msl_request_builder.py index 8f9764de3..4b601e98c 100644 --- a/resources/lib/services/msl/msl_request_builder.py +++ b/resources/lib/services/msl/msl_request_builder.py @@ -55,9 +55,9 @@ def build_request_data(url, params=None, echo=''): return request_data @common.time_execution(immediate=True) - def msl_request(self, data, esn): + def msl_request(self, data, esn, auth_data): """Create an encrypted MSL request""" - return (json.dumps(self._signed_header(esn)) + + return (json.dumps(self._signed_header(esn, auth_data)) + json.dumps(self._encrypted_chunk(data, esn))) @common.time_execution(immediate=True) @@ -69,15 +69,15 @@ def handshake_request(self, esn): 'authdata': {'identity': esn}}, 'headerdata': base64.standard_b64encode( - self._headerdata(is_handshake=True).encode('utf-8')).decode('utf-8'), + self._headerdata(auth_data={}, is_handshake=True).encode('utf-8')).decode('utf-8'), 'signature': '' }, sort_keys=True) payload = json.dumps(self._encrypted_chunk(envelope_payload=False)) return header + payload @common.time_execution(immediate=True) - def _signed_header(self, esn): - encryption_envelope = self.crypto.encrypt(self._headerdata(esn=esn), esn) + def _signed_header(self, esn, auth_data): + encryption_envelope = self.crypto.encrypt(self._headerdata(auth_data=auth_data, esn=esn), esn) return { 'headerdata': base64.standard_b64encode( encryption_envelope.encode('utf-8')).decode('utf-8'), @@ -85,7 +85,7 @@ def _signed_header(self, esn): 'mastertoken': self.crypto.mastertoken, } - def _headerdata(self, esn=None, compression=None, is_handshake=False): + def _headerdata(self, auth_data, esn=None, compression=None, is_handshake=False): """ Function that generates a MSL header dict :return: The base64 encoded JSON String of the header @@ -104,7 +104,7 @@ def _headerdata(self, esn=None, compression=None, is_handshake=False): header_data['keyrequestdata'] = self.crypto.key_request_data() else: header_data['sender'] = esn - self._add_auth_info(header_data) + self._add_auth_info(header_data, auth_data) return json.dumps(header_data) @@ -134,14 +134,33 @@ def decrypt_header_data(self, data, enveloped=True): return json.loads(self.crypto.decrypt(init_vector, cipher_text)) return header_data - def _add_auth_info(self, header_data): + def _add_auth_info(self, header_data, auth_data): """User authentication identifies the application user associated with a message""" - # Authentication with the user credentials - credentials = common.get_credentials() - header_data['userauthdata'] = { - 'scheme': 'EMAIL_PASSWORD', - 'authdata': { - 'email': credentials['email'], - 'password': credentials['password'] + # Warning: the user id token contains also contains the identity of the netflix profile + # therefore it is necessary to use the right user id token for the request + if auth_data.get('user_id_token'): + if auth_data['use_switch_profile']: + # The SWITCH_PROFILE is a custom Netflix MSL user authentication scheme + # that is needed for switching profile on MSL side + # works only combined with user id token and can not be used with all endpoints + # after use it you will get user id token of the profile specified in the response + header_data['userauthdata'] = { + 'scheme': 'SWITCH_PROFILE', + 'authdata': { + 'useridtoken': auth_data['user_id_token'], + 'profileguid': g.LOCAL_DB.get_active_profile_guid() + } + } + else: + # Authentication with user ID token containing the user identity (netflix profile) + header_data['useridtoken'] = auth_data['user_id_token'] + else: + # Authentication with the user credentials + credentials = common.get_credentials() + header_data['userauthdata'] = { + 'scheme': 'EMAIL_PASSWORD', + 'authdata': { + 'email': credentials['email'], + 'password': credentials['password'] + } } - } diff --git a/resources/lib/services/msl/msl_requests.py b/resources/lib/services/msl/msl_requests.py index 424e8ff98..8d59c4e38 100644 --- a/resources/lib/services/msl/msl_requests.py +++ b/resources/lib/services/msl/msl_requests.py @@ -21,7 +21,12 @@ from resources.lib.globals import g from resources.lib.services.msl.exceptions import MSLError from resources.lib.services.msl.msl_request_builder import MSLRequestBuilder -from resources.lib.services.msl.msl_utils import display_error_info, ENDPOINTS +from resources.lib.services.msl.msl_utils import display_error_info, generate_logblobs_params, EVENT_BIND, ENDPOINTS + +try: # Python 2 + from urllib import urlencode +except ImportError: # Python 3 + from urllib.parse import urlencode class MSLRequests(MSLRequestBuilder): @@ -30,6 +35,7 @@ def __init__(self, msl_data=None): super(MSLRequests, self).__init__() self.session = requests.session() self._load_msl_data(msl_data) + self.msl_switch_requested = False def _load_msl_data(self, msl_data): try: @@ -62,10 +68,29 @@ def perform_key_handshake(self, data=None): self.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) # Delete all the user id tokens (are correlated to the previous mastertoken) - # self.crypto.clear_user_id_tokens() + self.crypto.clear_user_id_tokens() common.debug('Key handshake successful') return True + def _get_owner_user_id_token(self): + """A way to get the user token id of owner profile""" + # In order to get a user id token of another (non-owner) profile you must make a request with SWITCH_PROFILE + # authentication scheme (a custom authentication for netflix), and this request can be directly included + # in the MSL manifest request. + # But in order to execute this switch profile, you need to have the user id token of the main (owner) profile. + # The only way (found to now) to get it immediately, is send a logblob event request, and save the + # user id token obtained in the response. + common.debug('Requesting logblog') + params = {'reqAttempt': 1, + 'reqPriority': 0, + 'reqName': EVENT_BIND} + url = ENDPOINTS['logblobs'] + '?' + urlencode(params).replace('%2F', '/') + response = self.chunked_request(url, + self.build_request_data('/logblob', generate_logblobs_params()), + g.get_esn(), + force_auth_credential=True) + common.debug('Response of logblob request: {}', response) + def _check_mastertoken_validity(self): """Return the mastertoken validity and executes a new key handshake when necessary""" if self.crypto.mastertoken: @@ -88,15 +113,56 @@ def _check_mastertoken_validity(self): return self._check_mastertoken_validity() return {'renewable': renewable, 'expired': expired} + def _check_user_id_token(self, disable_msl_switch, force_auth_credential=False): + """ + Performs user id token checks and return the auth data + checks: uid token validity, get if needed the owner uid token, set when use the switch + + :param: disable_msl_switch: to be used in requests that cannot make the switch + :param: force_auth_credential: force the use of authentication with credentials + :return: auth data that will be used in MSLRequestBuilder _add_auth_info + """ + # Warning: the user id token contains also contains the identity of the netflix profile + # therefore it is necessary to use the right user id token for the request + current_profile_guid = g.LOCAL_DB.get_active_profile_guid() + owner_profile_guid = g.LOCAL_DB.get_guid_owner_profile() + use_switch_profile = False + user_id_token = None + + if not force_auth_credential: + if current_profile_guid == owner_profile_guid: + # It is not necessary to get a token id because by default MSL it is associated to the main profile + # So you do not even need to run the MSL profile switch + user_id_token = self.crypto.get_user_id_token(current_profile_guid) + # user_id_token can return None when the add-on is installed from scratch, in this case will be used + # the authentication with the user credentials + else: + # The request must be executed from a non-owner profile + # Check if the token for the profile exist and valid + user_id_token = self.crypto.get_user_id_token(current_profile_guid) + if not user_id_token and not disable_msl_switch: + # If it is not there, first check if the main profile token exist and valid + use_switch_profile = True + user_id_token = self.crypto.get_user_id_token(owner_profile_guid) + # If it is not there, you must obtain it before making the MSL switch + if not user_id_token: + self._get_owner_user_id_token() + user_id_token = self.crypto.get_user_id_token(owner_profile_guid) + # Mark msl_switch_requested as True in order to make a bind event request + self.msl_switch_requested = True + return {'use_switch_profile': use_switch_profile, 'user_id_token': user_id_token} + @common.time_execution(immediate=True) - def chunked_request(self, endpoint, request_data, esn): + def chunked_request(self, endpoint, request_data, esn, disable_msl_switch=True, force_auth_credential=False): """Do a POST request and process the chunked response""" mt_validity = self._check_mastertoken_validity() + auth_data = self._check_user_id_token(disable_msl_switch, force_auth_credential) chunked_response = self._process_chunked_response( - self._post(endpoint, self.msl_request(request_data, esn)), - mt_validity['renewable'] if mt_validity else None) + self._post(endpoint, self.msl_request(request_data, esn, auth_data)), + mt_validity['renewable'] if mt_validity else None, + save_uid_token_to_owner=auth_data['user_id_token'] is None) return chunked_response['result'] @common.time_execution(immediate=True) @@ -112,7 +178,7 @@ def _post(self, endpoint, request_data): # pylint: disable=unused-argument @common.time_execution(immediate=True) - def _process_chunked_response(self, response, mt_renewable): + def _process_chunked_response(self, response, mt_renewable, save_uid_token_to_owner=False): """Parse and decrypt an encrypted chunked response. Raise an error if the response is plaintext json""" try: @@ -128,8 +194,13 @@ def _process_chunked_response(self, response, mt_renewable): # # Check if mastertoken is renewed # self.request_builder.crypto.compare_mastertoken(response['header']['mastertoken']) - # header_data = self.decrypt_header_data(response['header'].get('headerdata')) + header_data = self.decrypt_header_data(response['header'].get('headerdata')) + if 'useridtoken' in header_data: + # Save the user id token for the future msl requests + profile_guid = g.LOCAL_DB.get_guid_owner_profile() if save_uid_token_to_owner else\ + g.LOCAL_DB.get_active_profile_guid() + self.crypto.save_user_id_token(profile_guid, header_data['useridtoken']) # if 'keyresponsedata' in header_data: # common.debug('Found key handshake in response data') # # Update current mastertoken diff --git a/resources/lib/services/msl/msl_utils.py b/resources/lib/services/msl/msl_utils.py index 621a4452c..63dd839c9 100644 --- a/resources/lib/services/msl/msl_utils.py +++ b/resources/lib/services/msl/msl_utils.py @@ -9,10 +9,18 @@ """ from __future__ import absolute_import, division, unicode_literals +import json +import random +import re +import time from functools import wraps +import xbmcgui + import resources.lib.kodi.ui as ui from resources.lib import common +from resources.lib.database.db_utils import TABLE_SESSION +from resources.lib.globals import g from resources.lib.services.msl.exceptions import MSLError try: # Python 2 @@ -137,3 +145,65 @@ def _find_video_data(player_state, manifest): # Not found? raise Exception('build_media_tag: unable to find video data with codec: {}, width: {}, height: {}' .format(codec, width, height)) + + +def generate_logblobs_params(): + """Generate the initial log blog""" + # It seems that this log is sent when logging in to a profile the first time + # i think it is the easiest to reproduce, the others contain too much data + screen_size = str(xbmcgui.getScreenWidth()) + 'x' + str(xbmcgui.getScreenHeight()) + timestamp_utc = time.time() + timestamp = int(timestamp_utc * 1000) + client_ver = g.LOCAL_DB.get_value('asset_core', '', table=TABLE_SESSION) + app_id = int(time.time()) * 10000 + random.randint(1, 10001) # Should be used with all log requests + if client_ver: + result = re.search(r'-([0-9\.]+)\.js$', client_ver) + client_ver = result.groups()[0] + + # Here you have to enter only the real data, falsifying the data would cause repercussions in netflix server logs + # therefore since it is possible to exclude data, we avoid entering data that we do not have + blob = { + 'browserua': common.get_user_agent().replace(' ', '#'), + 'browserhref': 'https://www.netflix.com/browse', + # 'initstart': 988, + # 'initdelay': 268, + 'screensize': screen_size, # '1920x1080', + 'screenavailsize': screen_size, # '1920x1040', + 'clientsize': screen_size, # '1920x944', + # 'pt_navigationStart': -1880, + # 'pt_fetchStart': -1874, + # 'pt_secureConnectionStart': -1880, + # 'pt_requestStart': -1853, + # 'pt_domLoading': -638, + # 'm_asl_start': 990, + # 'm_stf_creat': 993, + # 'm_idb_open': 993, + # 'm_idb_succ': 1021, + # 'm_msl_load_no_data': 1059, + # 'm_asl_comp': 1256, + 'type': 'startup', + 'sev': 'info', + 'devmod': 'chrome-cadmium', + 'clver': client_ver, # e.g. '6.0021.220.051' + 'osplatform': g.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION), + 'osver': g.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION), + 'browsername': 'Chrome', + 'browserver': g.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION), + 'appLogSeqNum': 0, + 'uniqueLogId': common.get_random_uuid(), + 'appId': app_id, + 'esn': g.get_esn(), + 'lver': '', + # 'jssid': '15822792997793', # Same value of appId + # 'jsoffms': 1261, + 'clienttime': timestamp, + 'client_utc': timestamp_utc, + 'uiver': g.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION) + } + + blobs_container = { + 'entries': [blob] + } + blobs_dump = json.dumps(blobs_container) + blobs_dump = blobs_dump.replace('"', '\"').replace(' ', '').replace('#', ' ') + return {'logblobs': blobs_dump} From 33292cf6f08b8ab9ba9b0bca66a826bd8f069b64 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 11:38:50 +0100 Subject: [PATCH 04/11] Updated test stub --- test/xbmcgui.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/xbmcgui.py b/test/xbmcgui.py index 16fe70179..fc2a0376d 100644 --- a/test/xbmcgui.py +++ b/test/xbmcgui.py @@ -16,6 +16,16 @@ ALPHANUM_HIDE_INPUT = 1 +def getScreenHeight(): + """A stub implementation of the xbmcgui getScreenHeight() function""" + return 1080 + + +def getScreenWidth(): + """A stub implementation of the xbmcgui getScreenWidth() function""" + return 1920 + + class Control: """A reimplementation of the xbmcgui Control class""" From 36c6ba71dc7979c84e9435f0f63364a0daa57fb1 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 14:01:50 +0100 Subject: [PATCH 05/11] Various changes --- resources/lib/services/msl/events_handler.py | 4 +- resources/lib/services/msl/msl_handler.py | 69 +++++++++---------- .../lib/services/msl/msl_request_builder.py | 1 + resources/lib/services/msl/msl_requests.py | 9 +-- resources/lib/services/msl/msl_utils.py | 2 +- .../services/playback/stream_continuity.py | 3 +- 6 files changed, 42 insertions(+), 46 deletions(-) diff --git a/resources/lib/services/msl/events_handler.py b/resources/lib/services/msl/events_handler.py index a3c735ef0..5013c104f 100644 --- a/resources/lib/services/msl/events_handler.py +++ b/resources/lib/services/msl/events_handler.py @@ -117,7 +117,7 @@ def _process_event_request(self, event): 'reqName': 'events/{}'.format(event)} url = ENDPOINTS['events'] + '?' + urlencode(params).replace('%2F', '/') try: - response = self.chunked_request(url, event.request_data, g.get_esn()) + response = self.chunked_request(url, event.request_data, g.get_esn(), disable_msl_switch=False) event.set_response(response) break except Exception as exc: # pylint: disable=broad-except @@ -236,5 +236,5 @@ def _build_event_params(self, event_type, event_data, player_state, manifest): def get_manifest(videoid): """Get the manifest from cache""" - cache_identifier = g.get_esn() + '_' + videoid.value + cache_identifier = g.LOCAL_DB.get_active_profile_guid() + '_' + g.get_esn() + '_' + videoid.value return g.CACHE.get(cache.CACHE_MANIFESTS, cache_identifier, False) diff --git a/resources/lib/services/msl/msl_handler.py b/resources/lib/services/msl/msl_handler.py index ce76bcec4..9503fadd1 100644 --- a/resources/lib/services/msl/msl_handler.py +++ b/resources/lib/services/msl/msl_handler.py @@ -40,21 +40,20 @@ class MSLHandler(object): def __init__(self): super(MSLHandler, self).__init__() - self.request_builder = None + self.msl_requests = None try: msl_data = json.loads(common.load_file('msl_data.json')) common.info('Loaded MSL data from disk') except Exception: # pylint: disable=broad-except msl_data = None - self.request_builder = MSLRequests(msl_data) + self.msl_requests = MSLRequests(msl_data) - events_handler = EventsHandler(self.request_builder.chunked_request) - events_handler.start() + EventsHandler(self.msl_requests.chunked_request).start() common.register_slot( signal=common.Signals.ESN_CHANGED, - callback=self.request_builder.perform_key_handshake) + callback=self.msl_requests.perform_key_handshake) common.register_slot( signal=common.Signals.RELEASE_LICENSE, callback=self.release_license) @@ -63,8 +62,7 @@ def __init__(self): @common.time_execution(immediate=True) def load_manifest(self, viewable_id): """ - Loads the manifets for the given viewable_id and - returns a mpd-XML-Manifest + Loads the manifests for the given viewable_id and returns a mpd-XML-Manifest :param viewable_id: The id of of the viewable :return: MPD XML Manifest or False if no success @@ -79,23 +77,22 @@ def load_manifest(self, viewable_id): return self.__tranform_to_dash(manifest) def get_edge_manifest(self, viewable_id, chrome_manifest): - """Load a manifest with an EDGE ESN and replace playback_context and - drm_context""" + """Load a manifest with an EDGE ESN and replace playback_context and drm_context""" common.debug('Loading EDGE manifest') esn = g.get_edge_esn() common.debug('Switching MSL data to EDGE') - self.request_builder.perform_key_handshake(esn) + self.msl_requests.perform_key_handshake(esn) manifest = self._load_manifest(viewable_id, esn) manifest['playbackContextId'] = chrome_manifest['playbackContextId'] manifest['drmContextId'] = chrome_manifest['drmContextId'] common.debug('Successfully loaded EDGE manifest') common.debug('Resetting MSL data to Chrome') - self.request_builder.perform_key_handshake() + self.msl_requests.perform_key_handshake() return manifest @common.time_execution(immediate=True) def _load_manifest(self, viewable_id, esn): - cache_identifier = esn + '_' + unicode(viewable_id) + cache_identifier = g.LOCAL_DB.get_active_profile_guid() + '_' + esn + '_' + unicode(viewable_id) try: # The manifest must be requested once and maintained for its entire duration manifest = g.CACHE.get(cache.CACHE_MANIFESTS, cache_identifier, False) @@ -160,10 +157,10 @@ def _load_manifest(self, viewable_id, esn): 'preferAssistiveAudio': False } - manifest = self.request_builder.chunked_request(ENDPOINTS['manifest'], - self.request_builder.build_request_data('/manifest', params), - esn, - disable_msl_switch=False) + manifest = self.msl_requests.chunked_request(ENDPOINTS['manifest'], + self.msl_requests.build_request_data('/manifest', params), + esn, + disable_msl_switch=False) if common.is_debug_verbose(): # Save the manifest to disk as reference common.save_file('manifest.json', json.dumps(manifest).encode('utf-8')) @@ -179,6 +176,7 @@ def _load_manifest(self, viewable_id, esn): def get_license(self, challenge, sid): """ Requests and returns a license for the given challenge and sid + :param challenge: The base64 encoded challenge :param sid: The sid paired to the challenge :return: Base64 representation of the license key or False unsuccessful @@ -193,41 +191,37 @@ def get_license(self, challenge, sid): 'challengeBase64': challenge, 'xid': xid }] - response = self.request_builder.chunked_request(ENDPOINTS['license'], - self.request_builder.build_request_data(self.last_license_url, - params, - 'sessionId'), - g.get_esn()) + response = self.msl_requests.chunked_request(ENDPOINTS['license'], + self.msl_requests.build_request_data(self.last_license_url, + params, + 'sessionId'), + g.get_esn()) # This xid must be used for any future request, until playback stops g.LOCAL_DB.set_value('xid', xid, TABLE_SESSION) self.last_license_session_id = sid self.last_license_release_url = response[0]['links']['releaseLicense']['href'] - if self.request_builder.msl_switch_requested: - self.request_builder.msl_switch_requested = False + if self.msl_requests.msl_switch_requested: + self.msl_requests.msl_switch_requested = False self.bind_events() return response[0]['licenseResponseBase64'] def bind_events(self): - """ - Bind events - """ + """Bind events""" # I don't know the real purpose of its use, it seems to be requested after the license and before starting # playback, and only the first time after a switch, # in the response you can also understand if the msl switch has worked common.debug('Requesting bind events') - response = self.request_builder.chunked_request(ENDPOINTS['events'], - self.request_builder.build_request_data('/bind', {}), - g.get_esn(), - disable_msl_switch=False) + response = self.msl_requests.chunked_request(ENDPOINTS['events'], + self.msl_requests.build_request_data('/bind', {}), + g.get_esn(), + disable_msl_switch=False) common.debug('Bind events response: {}', response) @display_error_info @common.time_execution(immediate=True) def release_license(self, data=None): # pylint: disable=unused-argument - """ - Release the server license - """ + """Release the server license""" common.debug('Requesting releasing license') params = [{ @@ -239,9 +233,9 @@ def release_license(self, data=None): # pylint: disable=unused-argument 'echo': 'sessionId' }] - response = self.request_builder.chunked_request(ENDPOINTS['license'], - self.request_builder.build_request_data('/bundle', params), - g.get_esn()) + response = self.msl_requests.chunked_request(ENDPOINTS['license'], + self.msl_requests.build_request_data('/bundle', params), + g.get_esn()) common.debug('License release response: {}', response) @common.time_execution(immediate=True) @@ -253,7 +247,6 @@ def __tranform_to_dash(self, manifest): def has_1080p(manifest): - """Return True if any of the video tracks in manifest have a 1080p profile - available, else False""" + """Return True if any of the video tracks in manifest have a 1080p profile available, else False""" return any(video['width'] >= 1920 for video in manifest['videoTracks'][0]['downloadables']) diff --git a/resources/lib/services/msl/msl_request_builder.py b/resources/lib/services/msl/msl_request_builder.py index 4b601e98c..1c66a4ce4 100644 --- a/resources/lib/services/msl/msl_request_builder.py +++ b/resources/lib/services/msl/msl_request_builder.py @@ -35,6 +35,7 @@ class MSLRequestBuilder(object): """Provides mechanisms to create MSL requests""" + def __init__(self): self.current_message_id = None self.rndm = random.SystemRandom() diff --git a/resources/lib/services/msl/msl_requests.py b/resources/lib/services/msl/msl_requests.py index 8d59c4e38..267b88bee 100644 --- a/resources/lib/services/msl/msl_requests.py +++ b/resources/lib/services/msl/msl_requests.py @@ -2,7 +2,8 @@ """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) Copyright (C) 2018 Caphm (original implementation module) - MSL request building + Copyright (C) 2020 Stefano Gottardo + MSL requests SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. @@ -30,6 +31,7 @@ class MSLRequests(MSLRequestBuilder): + """Provides methods to make MSL requests""" def __init__(self, msl_data=None): super(MSLRequests, self).__init__() @@ -44,6 +46,7 @@ def _load_msl_data(self, msl_data): # Add-on just installed, the service starts but there is no esn if g.get_esn(): + # This is also done here only try to speed up the loading of manifest self._check_mastertoken_validity() except MSLError: raise @@ -62,7 +65,6 @@ def perform_key_handshake(self, data=None): return False common.debug('Performing key handshake. ESN: {}', esn) - response = _process_json_response(self._post(ENDPOINTS['manifest'], self.handshake_request(esn))) header_data = self.decrypt_header_data(response['headerdata'], False) self.crypto.parse_key_response(header_data, not common.is_edge_esn(esn)) @@ -158,6 +160,7 @@ def chunked_request(self, endpoint, request_data, esn, disable_msl_switch=True, mt_validity = self._check_mastertoken_validity() auth_data = self._check_user_id_token(disable_msl_switch, force_auth_credential) + common.debug('Chunked request will be executed with auth data: {}', auth_data) chunked_response = self._process_chunked_response( self._post(endpoint, self.msl_request(request_data, esn, auth_data)), @@ -165,7 +168,6 @@ def chunked_request(self, endpoint, request_data, esn, disable_msl_switch=True, save_uid_token_to_owner=auth_data['user_id_token'] is None) return chunked_response['result'] - @common.time_execution(immediate=True) def _post(self, endpoint, request_data): """Execute a post request""" common.debug('Executing POST request to {}', endpoint) @@ -282,5 +284,4 @@ def _decrypt_chunks(chunks, crypto): data = base64.standard_b64decode(data).decode('utf-8') decrypted_payload += data - return json.loads(decrypted_payload) diff --git a/resources/lib/services/msl/msl_utils.py b/resources/lib/services/msl/msl_utils.py index 63dd839c9..a1c574ebf 100644 --- a/resources/lib/services/msl/msl_utils.py +++ b/resources/lib/services/msl/msl_utils.py @@ -2,7 +2,7 @@ """ Copyright (C) 2017 Sebastian Golasch (plugin.video.netflix) Copyright (C) 2019 Stefano Gottardo - @CastagnaIT (original implementation module) - Msl utils + MSL utils SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. diff --git a/resources/lib/services/playback/stream_continuity.py b/resources/lib/services/playback/stream_continuity.py index 9f06567ea..a9fb9330c 100644 --- a/resources/lib/services/playback/stream_continuity.py +++ b/resources/lib/services/playback/stream_continuity.py @@ -240,7 +240,8 @@ def _show_only_forced_subtitle(self): # --- ONLY FOR KODI VERSION 18 --- # NOTE: With Kodi 18 it is not possible to read the properties of the streams # so the only possible way is to read the data from the manifest file - cache_identifier = g.get_esn() + '_' + self.current_videoid.value + cache_identifier = (g.LOCAL_DB.get_active_profile_guid() + '_' + + g.get_esn() + '_' + self.current_videoid.value) manifest_data = g.CACHE.get(CACHE_MANIFESTS, cache_identifier, False) common.fix_locale_languages(manifest_data['timedtexttracks']) if not any(text_track.get('isForcedNarrative', False) is True and From f88e831aa9743294a7089c26b010cc0f67280f12 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 14:52:01 +0100 Subject: [PATCH 06/11] Enabled sync watched status with all profiles --- resources/language/resource.language.en_gb/strings.po | 2 +- resources/lib/kodi/context_menu.py | 2 +- resources/lib/kodi/infolabels.py | 5 +---- resources/lib/navigation/player.py | 5 ++--- resources/lib/services/playback/progress_manager.py | 10 ---------- 5 files changed, 5 insertions(+), 19 deletions(-) diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 0a83af8de..ab16ca08b 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -965,7 +965,7 @@ msgid "Install Up Next add-on" msgstr "" msgctxt "#30235" -msgid "[WIP] Send/Receive progress and watched status of the videos (works only with the main profile)" +msgid "[WIP] Synchronize the watched status of the videos with Netflix" msgstr "" msgctxt "#30236" diff --git a/resources/lib/kodi/context_menu.py b/resources/lib/kodi/context_menu.py index e43f899e5..8cf6775c1 100644 --- a/resources/lib/kodi/context_menu.py +++ b/resources/lib/kodi/context_menu.py @@ -102,7 +102,7 @@ def generate_context_menu_items(videoid): if videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]: # Add menu to allow change manually the watched status when progress manager is enabled - if g.ADDON.getSettingBool('ProgressManager_enabled') and g.LOCAL_DB.get_profile_config('isAccountOwner', False): + if g.ADDON.getSettingBool('ProgressManager_enabled'): items.insert(0, _ctx_item('change_watched_status', videoid)) return items diff --git a/resources/lib/kodi/infolabels.py b/resources/lib/kodi/infolabels.py index 699bcd019..2d6e906e8 100644 --- a/resources/lib/kodi/infolabels.py +++ b/resources/lib/kodi/infolabels.py @@ -351,10 +351,7 @@ def _colorize_title(text, color, remove_color=False): def _set_progress_status(list_item, video_data, infos): """Check and set progress status (watched and resume)""" - if not g.ADDON.getSettingBool('ProgressManager_enabled') or \ - not g.LOCAL_DB.get_profile_config('isAccountOwner', False): - # Currently due to a unknown problem, it is not possible to communicate MSL data to the right selected - # profile other than the owner profile + if not g.ADDON.getSettingBool('ProgressManager_enabled'): return video_id = video_data['summary']['id'] diff --git a/resources/lib/navigation/player.py b/resources/lib/navigation/player.py index ceabb9e0a..d7cbc9755 100644 --- a/resources/lib/navigation/player.py +++ b/resources/lib/navigation/player.py @@ -89,10 +89,9 @@ def play(videoid): return if index_selected == 1: resume_position = None - elif g.ADDON.getSettingBool('ProgressManager_enabled') and g.LOCAL_DB.get_profile_config('isAccountOwner', False): + elif (g.ADDON.getSettingBool('ProgressManager_enabled') and + videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]): # To now we have this limits: - # - enabled only if the owner profile is used. Currently due to a unknown problem, - # it is not possible to communicate MSL data to the right selected profile # - enabled only with items played inside the addon then not Kodi library, need impl. JSON-RPC lib update code event_data = _get_event_data(videoid) event_data['videoid'] = videoid.to_dict() diff --git a/resources/lib/services/playback/progress_manager.py b/resources/lib/services/playback/progress_manager.py index ee4cc5a7a..acf03375c 100644 --- a/resources/lib/services/playback/progress_manager.py +++ b/resources/lib/services/playback/progress_manager.py @@ -12,7 +12,6 @@ from xbmcgui import Window import resources.lib.common as common -from resources.lib.globals import g from resources.lib.services.msl.msl_utils import EVENT_START, EVENT_ENGAGE, EVENT_STOP, EVENT_KEEP_ALIVE from .action_manager import PlaybackActionManager @@ -32,15 +31,6 @@ def __init__(self): # pylint: disable=super-on-old-class self.window_cls = Window(10000) # Kodi home window def _initialize(self, data): - if not g.LOCAL_DB.get_profile_config('isAccountOwner', False): - # Currently due to a unknown problem, it is not possible to communicate MSL data to the right selected - # profile other than the owner profile - self.enabled = False - return - videoid = common.VideoId.from_dict(data['videoid']) - if videoid.mediatype not in [common.VideoId.MOVIE, common.VideoId.EPISODE]: - self.enabled = False - return if not data['event_data']: common.warn('ProgressManager: disabled due to no event data') self.enabled = False From 8984455d5a754771b5d2997ff89519603add56b1 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 14:52:47 +0100 Subject: [PATCH 07/11] Fixed wrong watched status threshold --- resources/lib/kodi/infolabels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/kodi/infolabels.py b/resources/lib/kodi/infolabels.py index 2d6e906e8..ac4ff8e89 100644 --- a/resources/lib/kodi/infolabels.py +++ b/resources/lib/kodi/infolabels.py @@ -368,7 +368,7 @@ def _set_progress_status(list_item, video_data, infos): if not video_data.get('creditsOffset'): # NOTE shakti 'creditsOffset' tag not exists on video type 'movie', # then simulate the default Kodi playcount behaviour (playcountminimumpercent) - watched_threshold = video_data['runtime'] - (video_data['runtime'] / 100 * 90) + watched_threshold = video_data['runtime'] / 100 * 90 else: watched_threshold = video_data['creditsOffset'] From 0dd4fd88c15eb55d25eecb8fbe68319c68906a58 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 16:35:55 +0100 Subject: [PATCH 08/11] Added required info from react context --- resources/lib/api/website.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/resources/lib/api/website.py b/resources/lib/api/website.py index de0b854b7..ff57417da 100644 --- a/resources/lib/api/website.py +++ b/resources/lib/api/website.py @@ -47,7 +47,12 @@ # 'ichnaea_log': 'models/serverDefs/data/ICHNAEA_ROOT', can be for XSS attacks? 'api_endpoint_root_url': 'models/serverDefs/data/API_ROOT', 'api_endpoint_url': 'models/playerModel/data/config/ui/initParams/apiUrl', - 'request_id': 'models/serverDefs/data/requestId' + 'request_id': 'models/serverDefs/data/requestId', + 'asset_core': 'models/playerModel/data/config/core/assets/core', + 'ui_version': 'models/playerModel/data/config/ui/initParams/uiVersion', + 'browser_info_version': 'models/browserInfo/data/version', + 'browser_info_os_name': 'models/browserInfo/data/os/name', + 'browser_info_os_version': 'models/browserInfo/data/os/version', } PAGE_ITEM_ERROR_CODE = 'models/flow/data/fields/errorCode/value' From 218d4d7eb3898f788408eec5d93e2e48f7dbcdc3 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sat, 22 Feb 2020 18:22:18 +0100 Subject: [PATCH 09/11] Fixed upgrade fix on install from scratch --- resources/lib/upgrade_controller.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/lib/upgrade_controller.py b/resources/lib/upgrade_controller.py index a4e2b81e6..0a8090564 100644 --- a/resources/lib/upgrade_controller.py +++ b/resources/lib/upgrade_controller.py @@ -86,7 +86,9 @@ def _perform_shared_db_changes(current_version, upgrade_to_version): # Init fix from resources.lib.common import is_minimum_version service_previous_ver = g.LOCAL_DB.get_value('service_previous_version', None) - if current_version is None and not is_minimum_version(service_previous_ver, '0.17.0'): + if service_previous_ver is not None and\ + current_version is None and\ + not is_minimum_version(service_previous_ver, '0.17.0'): current_version = '0.1' # End fix From 25b2fa3de689a8b69157182298253ce603ba2727 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sun, 23 Feb 2020 10:19:50 +0100 Subject: [PATCH 10/11] Fixed debug output --- resources/lib/services/msl/android_crypto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/lib/services/msl/android_crypto.py b/resources/lib/services/msl/android_crypto.py index bbe6a1860..18f551e90 100644 --- a/resources/lib/services/msl/android_crypto.py +++ b/resources/lib/services/msl/android_crypto.py @@ -42,8 +42,8 @@ def __init__(self): 'version': self.crypto_session.GetPropertyString('version'), 'system_id': self.crypto_session.GetPropertyString('systemId'), # 'device_unique_id': self.crypto_session.GetPropertyByteArray('deviceUniqueId') - 'hdcp_level': self.crypto_session.GetPropertyString('hdcpLevel'), - 'hdcp_level_max': self.crypto_session.GetPropertyString('maxHdcpLevel'), + 'hdcp_level': self.crypto_session.GetPropertyString('hdcpLevel'), #WTF perchè non viene fuori??? serve volontario test + 'hdcp_level_max': self.crypto_session.GetPropertyString('maxHdcpLevel'), #WTF perchè non viene fuori??? serve volontario test 'security_level': self.crypto_session.GetPropertyString('securityLevel') } @@ -64,8 +64,8 @@ def __init__(self): else: common.warn('Widevine CryptoSession system id not obtained!') common.debug('Widevine CryptoSession security level: {}', drm_info['security_level']) - common.debug('Widevine CryptoSession current hdcp level', drm_info['hdcp_level']) - common.debug('Widevine CryptoSession max hdcp level supported', drm_info['hdcp_level_max']) + common.debug('Widevine CryptoSession current hdcp level: {}', drm_info['hdcp_level']) + common.debug('Widevine CryptoSession max hdcp level supported: {}', drm_info['hdcp_level_max']) common.debug('Widevine CryptoSession algorithms: {}', self.crypto_session.GetPropertyString('algorithms')) def load_crypto_session(self, msl_data=None): From cceefa6ab3781bf785ea3134641109dcf207ab36 Mon Sep 17 00:00:00 2001 From: CastagnaIT Date: Sun, 23 Feb 2020 10:26:35 +0100 Subject: [PATCH 11/11] Improved watched threshold --- resources/lib/kodi/infolabels.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/resources/lib/kodi/infolabels.py b/resources/lib/kodi/infolabels.py index ac4ff8e89..5410f5104 100644 --- a/resources/lib/kodi/infolabels.py +++ b/resources/lib/kodi/infolabels.py @@ -365,11 +365,10 @@ def _set_progress_status(list_item, video_data, infos): # seem not respect really if a video is watched to the end or this tag have other purposes # to now, the only way to know if a video is watched is compare the bookmarkPosition with creditsOffset value - if not video_data.get('creditsOffset'): - # NOTE shakti 'creditsOffset' tag not exists on video type 'movie', - # then simulate the default Kodi playcount behaviour (playcountminimumpercent) - watched_threshold = video_data['runtime'] / 100 * 90 - else: + # NOTE shakti 'creditsOffset' tag not exists on video type 'movie', + # then simulate the default Kodi playcount behaviour (playcountminimumpercent) + watched_threshold = video_data['runtime'] / 100 * 90 + if video_data.get('creditsOffset') and video_data['creditsOffset'] < watched_threshold: watched_threshold = video_data['creditsOffset'] # NOTE shakti 'bookmarkPosition' tag when it is not set have -1 value