diff --git a/python/pyspark/broadcast.py b/python/pyspark/broadcast.py index 74dee1420754a..e04f1c3fb63d5 100644 --- a/python/pyspark/broadcast.py +++ b/python/pyspark/broadcast.py @@ -21,6 +21,7 @@ from tempfile import NamedTemporaryFile from pyspark.cloudpickle import print_exec +from pyspark.serializers import protocol if sys.version < '3': import cPickle as pickle @@ -78,7 +79,7 @@ def __init__(self, sc=None, value=None, pickle_registry=None, path=None): def dump(self, value, f): try: - pickle.dump(value, f, 2) + pickle.dump(value, f, protocol) except pickle.PickleError: raise except Exception as e: diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py index 2a1326947f4f5..d2a65351433bf 100644 --- a/python/pyspark/serializers.py +++ b/python/pyspark/serializers.py @@ -64,7 +64,7 @@ from itertools import izip as zip else: import pickle - protocol = 3 + protocol = min(pickle.HIGHEST_PROTOCOL, 4) xrange = range from pyspark import cloudpickle