Skip to content

[SPARK-28586][INFRA] Make merge-spark-pr script compatible with Python 3 #25322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HyukjinKwon wants to merge 1 commit into from
Closed

[SPARK-28586][INFRA] Make merge-spark-pr script compatible with Python 3 #25322

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 28 additions & 22 deletions dev/merge_spark_pr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,22 @@
import subprocess
import sys
import traceback
import urllib2
if sys.version < '3':
input = raw_input # noqa
from urllib2 import urlopen
from urllib2 import Request
from urllib2 import HTTPError
else:
from urllib.request import urlopen
from urllib.request import Request
from urllib.error import HTTPError

try:
import jira.client
JIRA_IMPORTED = True
except ImportError:
JIRA_IMPORTED = False

if sys.version < '3':
input = raw_input # noqa

# Location of your Spark git development area
SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd())
# Remote name which points to the Gihub site
Expand Down Expand Up @@ -69,11 +74,11 @@

def get_json(url):
try:
request = urllib2.Request(url)
request = Request(url)
if GITHUB_OAUTH_KEY:
request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
return json.load(urllib2.urlopen(request))
except urllib2.HTTPError as e:
return json.load(urlopen(request))
except HTTPError as e:
if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == '0':
print("Exceeded the GitHub API rate limit; see the instructions in " +
"dev/merge_spark_pr.py to configure an OAuth token for making authenticated " +
Expand All @@ -92,9 +97,9 @@ def fail(msg):
def run_cmd(cmd):
print(cmd)
if isinstance(cmd, list):
return subprocess.check_output(cmd)
return subprocess.check_output(cmd).decode(sys.stdout.encoding)
else:
return subprocess.check_output(cmd.split(" "))
return subprocess.check_output(cmd.split(" ")).decode(sys.stdout.encoding)


def continue_maybe(prompt):
Expand All @@ -110,7 +115,7 @@ def clean_up():

branches = run_cmd("git branch").replace(" ", "").split("\n")

for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):
for branch in list(filter(lambda x: x.startswith(BRANCH_PREFIX), branches)):
print("Deleting local branch %s" % branch)
run_cmd("git branch -D %s" % branch)

Expand Down Expand Up @@ -237,7 +242,7 @@ def fix_version_from_branch(branch, versions):
return versions[0]
else:
branch_ver = branch.replace("branch-", "")
return filter(lambda x: x.name.startswith(branch_ver), versions)[-1]
return list(filter(lambda x: x.name.startswith(branch_ver), versions))[-1]


def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
Expand Down Expand Up @@ -272,11 +277,12 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""):

versions = asf_jira.project_versions("SPARK")
versions = sorted(versions, key=lambda x: x.name, reverse=True)
versions = filter(lambda x: x.raw['released'] is False, versions)
versions = list(filter(lambda x: x.raw['released'] is False, versions))
# Consider only x.y.z versions
versions = filter(lambda x: re.match(r'\d+\.\d+\.\d+', x.name), versions)
versions = list(filter(lambda x: re.match(r'\d+\.\d+\.\d+', x.name), versions))

default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)
default_fix_versions = list(map(
lambda x: fix_version_from_branch(x, versions).name, merge_branches))
for v in default_fix_versions:
# Handles the case where we have forked a release branch but not yet made the release.
# In this case, if the PR is committed to the master branch and the release branch, we
Expand All @@ -286,10 +292,10 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
if patch == "0":
previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
if previous in default_fix_versions:
default_fix_versions = filter(lambda x: x != v, default_fix_versions)
default_fix_versions = list(filter(lambda x: x != v, default_fix_versions))
default_fix_versions = ",".join(default_fix_versions)

available_versions = set(map(lambda v: v.name, versions))
available_versions = set(list(map(lambda v: v.name, versions)))
while True:
try:
fix_versions = input(
Expand All @@ -309,12 +315,12 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
print("Error setting fix version(s), try again (or leave blank and fix manually)")

def get_version_json(version_str):
return filter(lambda v: v.name == version_str, versions)[0].raw
return list(filter(lambda v: v.name == version_str, versions))[0].raw

jira_fix_versions = map(lambda v: get_version_json(v), fix_versions)
jira_fix_versions = list(map(lambda v: get_version_json(v), fix_versions))

resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0]
resolution = filter(lambda r: r.raw['name'] == "Fixed", asf_jira.resolutions())[0]
resolve = list(filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id)))[0]
resolution = list(filter(lambda r: r.raw['name'] == "Fixed", asf_jira.resolutions()))[0]
asf_jira.transition_issue(
jira_id, resolve["id"], fixVersions=jira_fix_versions,
comment=comment, resolution={'id': resolution.raw['id']})
Expand All @@ -330,7 +336,7 @@ def choose_jira_assignee(issue, asf_jira):
while True:
try:
reporter = issue.fields.reporter
commentors = map(lambda x: x.author, issue.fields.comment.comments)
commentors = list(map(lambda x: x.author, issue.fields.comment.comments))
candidates = set(commentors)
candidates.add(reporter)
candidates = list(candidates)
Expand Down Expand Up @@ -457,7 +463,7 @@ def main():
continue_maybe("The env-vars JIRA_USERNAME and/or JIRA_PASSWORD are not set. Continue?")

branches = get_json("%s/branches" % GITHUB_API_BASE)
branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches])
branch_names = list(filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches]))
# Assumes branch names can be sorted lexicographically
latest_branch = sorted(branch_names, reverse=True)[0]

Expand Down