From f660fa8fea399f6595326b099f329d82de0e3124 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Wed, 9 Dec 2020 09:45:30 +0100 Subject: [PATCH 01/18] init --- .../site/assets/icons/community/bee-icon.svg | 38 ++++ .../site/assets/icons/community/box-icon.svg | 22 +++ .../assets/icons/community/calendar-icon.svg | 23 +++ .../assets/icons/community/contact-us/bug.svg | 22 +++ .../icons/community/contact-us/discussion.svg | 23 +++ .../icons/community/contact-us/gitArrows.svg | 23 +++ .../icons/community/contact-us/knot.svg | 22 +++ .../icons/community/contact-us/messages.svg | 22 +++ .../community/contact-us/notification.svg | 22 +++ .../community/contact-us/question-mark.svg | 23 +++ .../assets/icons/community/envelope-icon.svg | 23 +++ .../assets/icons/community/message-icon.svg | 23 +++ .../site/assets/icons/install-button-icon.svg | 22 +++ .../site/assets/icons/navbar-arrow-icon.svg | 23 +++ .../icons/navbar-documentation-icon.svg | 22 +++ website/www/site/assets/icons/quote-icon.svg | 22 +++ website/www/site/assets/scss/_cards.sass | 69 ------- website/www/site/assets/scss/_global.sass | 17 +- website/www/site/assets/scss/_graphic.sass | 2 + .../www/site/assets/scss/_hero-mobile.scss | 62 ++++++ website/www/site/assets/scss/_hero.sass | 156 --------------- website/www/site/assets/scss/_hero.scss | 128 +++++++++++++ .../www/site/assets/scss/_keen-slider.scss | 40 ++++ ...ble-wrapper.sass => _list-with-icons.scss} | 31 ++- .../www/site/assets/scss/_navbar-desktop.scss | 180 ++++++++++++++++++ .../{_navbar.sass => _navbar-mobile.sass} | 41 ++-- website/www/site/assets/scss/_quotes.scss | 151 +++++++++++++++ .../www/site/assets/scss/_section-nav.sass | 8 +- .../www/site/assets/scss/_table-wrapper.scss | 103 ++++++++++ website/www/site/assets/scss/_typography.scss | 74 ++++++- website/www/site/assets/scss/_vars.sass | 4 + .../site/assets/scss/bootstrap/_navbar.scss | 2 +- website/www/site/assets/scss/main.scss | 15 +- .../www/site/content/en/community/_index.md | 28 +++ .../site/content/en/community/contact-us.md | 36 ++-- .../site/content/en/community/powered-by.md | 2 + .../www/site/content/en/contribute/_index.md | 16 +- .../sql/extensions/create-external-table.md | 132 +++++++++++++ .../en/documentation/io/developing-io-java.md | 3 + .../io/developing-io-overview.md | 80 ++++---- .../documentation/io/developing-io-python.md | 3 + .../en/documentation/programming-guide.md | 2 +- .../documentation/sdks/python-dependencies.md | 12 +- .../content/en/get-started/quickstart-py.md | 4 +- .../en/get-started/wordcount-example.md | 2 +- website/www/site/data/en/community_list.yaml | 27 +++ website/www/site/data/en/contact_us.yaml | 41 ++++ website/www/site/data/en/quotes.yaml | 23 +++ website/www/site/i18n/home/en.yaml | 4 - .../en/cards.yaml => i18n/home/hero/en.yaml} | 14 +- website/www/site/i18n/home/quotes/en.yaml | 14 ++ website/www/site/i18n/navbar/en.yaml | 4 +- website/www/site/layouts/_default/baseof.html | 4 +- .../www/site/layouts/community/baseof.html | 54 +++--- website/www/site/layouts/index.html | 121 +++++------- website/www/site/layouts/partials/head.html | 2 +- website/www/site/layouts/partials/header.html | 84 ++++++-- .../site/layouts/partials/hooks/body-end.html | 17 ++ .../layouts/partials/quotes/quote-mobile.html | 21 ++ .../site/layouts/partials/quotes/quote.html | 21 ++ .../partials/section-menu/en/community.html | 2 +- .../layouts/shortcodes/list_with_icons.html | 29 +++ .../layouts/shortcodes/table_with_icons.html | 51 +++++ .../static/images/beam_logo_navbar_mobile.png | Bin 0 -> 1497 bytes .../site/static/images/quote-paypal-logo.png | Bin 0 -> 16047 bytes .../www/site/static/js/hero/hero-desktop.js | 21 ++ .../www/site/static/js/hero/hero-mobile.js | 21 ++ .../site/static/js/hero/lottie-light.min.js | 18 ++ website/www/site/static/js/keen-slider.min.js | 15 ++ website/www/site/static/js/quotes-slider.js | 42 ++++ website/www/site/static/js/section-nav.js | 2 +- 71 files changed, 1969 insertions(+), 461 deletions(-) create mode 100644 website/www/site/assets/icons/community/bee-icon.svg create mode 100644 website/www/site/assets/icons/community/box-icon.svg create mode 100644 website/www/site/assets/icons/community/calendar-icon.svg create mode 100644 website/www/site/assets/icons/community/contact-us/bug.svg create mode 100644 website/www/site/assets/icons/community/contact-us/discussion.svg create mode 100644 website/www/site/assets/icons/community/contact-us/gitArrows.svg create mode 100644 website/www/site/assets/icons/community/contact-us/knot.svg create mode 100644 website/www/site/assets/icons/community/contact-us/messages.svg create mode 100644 website/www/site/assets/icons/community/contact-us/notification.svg create mode 100644 website/www/site/assets/icons/community/contact-us/question-mark.svg create mode 100644 website/www/site/assets/icons/community/envelope-icon.svg create mode 100644 website/www/site/assets/icons/community/message-icon.svg create mode 100644 website/www/site/assets/icons/install-button-icon.svg create mode 100644 website/www/site/assets/icons/navbar-arrow-icon.svg create mode 100644 website/www/site/assets/icons/navbar-documentation-icon.svg create mode 100644 website/www/site/assets/icons/quote-icon.svg delete mode 100644 website/www/site/assets/scss/_cards.sass create mode 100644 website/www/site/assets/scss/_hero-mobile.scss delete mode 100644 website/www/site/assets/scss/_hero.sass create mode 100644 website/www/site/assets/scss/_hero.scss create mode 100644 website/www/site/assets/scss/_keen-slider.scss rename website/www/site/assets/scss/{_table-wrapper.sass => _list-with-icons.scss} (60%) create mode 100644 website/www/site/assets/scss/_navbar-desktop.scss rename website/www/site/assets/scss/{_navbar.sass => _navbar-mobile.sass} (73%) create mode 100644 website/www/site/assets/scss/_quotes.scss create mode 100644 website/www/site/assets/scss/_table-wrapper.scss create mode 100644 website/www/site/content/en/community/_index.md create mode 100644 website/www/site/data/en/community_list.yaml create mode 100644 website/www/site/data/en/contact_us.yaml create mode 100644 website/www/site/data/en/quotes.yaml rename website/www/site/{data/en/cards.yaml => i18n/home/hero/en.yaml} (60%) create mode 100644 website/www/site/i18n/home/quotes/en.yaml create mode 100644 website/www/site/layouts/partials/hooks/body-end.html create mode 100644 website/www/site/layouts/partials/quotes/quote-mobile.html create mode 100644 website/www/site/layouts/partials/quotes/quote.html create mode 100644 website/www/site/layouts/shortcodes/list_with_icons.html create mode 100644 website/www/site/layouts/shortcodes/table_with_icons.html create mode 100644 website/www/site/static/images/beam_logo_navbar_mobile.png create mode 100644 website/www/site/static/images/quote-paypal-logo.png create mode 100644 website/www/site/static/js/hero/hero-desktop.js create mode 100644 website/www/site/static/js/hero/hero-mobile.js create mode 100644 website/www/site/static/js/hero/lottie-light.min.js create mode 100644 website/www/site/static/js/keen-slider.min.js create mode 100644 website/www/site/static/js/quotes-slider.js diff --git a/website/www/site/assets/icons/community/bee-icon.svg b/website/www/site/assets/icons/community/bee-icon.svg new file mode 100644 index 000000000000..d04da17d1006 --- /dev/null +++ b/website/www/site/assets/icons/community/bee-icon.svg @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/community/box-icon.svg b/website/www/site/assets/icons/community/box-icon.svg new file mode 100644 index 000000000000..572896e70ec2 --- /dev/null +++ b/website/www/site/assets/icons/community/box-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/community/calendar-icon.svg b/website/www/site/assets/icons/community/calendar-icon.svg new file mode 100644 index 000000000000..18229ffe4836 --- /dev/null +++ b/website/www/site/assets/icons/community/calendar-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/bug.svg b/website/www/site/assets/icons/community/contact-us/bug.svg new file mode 100644 index 000000000000..62267024ea72 --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/bug.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/discussion.svg b/website/www/site/assets/icons/community/contact-us/discussion.svg new file mode 100644 index 000000000000..dc9c94e95d50 --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/discussion.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/gitArrows.svg b/website/www/site/assets/icons/community/contact-us/gitArrows.svg new file mode 100644 index 000000000000..30a857d94adc --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/gitArrows.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/knot.svg b/website/www/site/assets/icons/community/contact-us/knot.svg new file mode 100644 index 000000000000..e47af8cd8ca2 --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/knot.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/messages.svg b/website/www/site/assets/icons/community/contact-us/messages.svg new file mode 100644 index 000000000000..43806ca5bc3a --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/messages.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/notification.svg b/website/www/site/assets/icons/community/contact-us/notification.svg new file mode 100644 index 000000000000..3b2976ad87ff --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/notification.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/community/contact-us/question-mark.svg b/website/www/site/assets/icons/community/contact-us/question-mark.svg new file mode 100644 index 000000000000..bb230303a16b --- /dev/null +++ b/website/www/site/assets/icons/community/contact-us/question-mark.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/community/envelope-icon.svg b/website/www/site/assets/icons/community/envelope-icon.svg new file mode 100644 index 000000000000..6ced3b9edd59 --- /dev/null +++ b/website/www/site/assets/icons/community/envelope-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/community/message-icon.svg b/website/www/site/assets/icons/community/message-icon.svg new file mode 100644 index 000000000000..466af8040595 --- /dev/null +++ b/website/www/site/assets/icons/community/message-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/install-button-icon.svg b/website/www/site/assets/icons/install-button-icon.svg new file mode 100644 index 000000000000..39c3ffcfbd31 --- /dev/null +++ b/website/www/site/assets/icons/install-button-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/navbar-arrow-icon.svg b/website/www/site/assets/icons/navbar-arrow-icon.svg new file mode 100644 index 000000000000..78724e930331 --- /dev/null +++ b/website/www/site/assets/icons/navbar-arrow-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/navbar-documentation-icon.svg b/website/www/site/assets/icons/navbar-documentation-icon.svg new file mode 100644 index 000000000000..11c165b0feba --- /dev/null +++ b/website/www/site/assets/icons/navbar-documentation-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/quote-icon.svg b/website/www/site/assets/icons/quote-icon.svg new file mode 100644 index 000000000000..0a32eea254ad --- /dev/null +++ b/website/www/site/assets/icons/quote-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/scss/_cards.sass b/website/www/site/assets/scss/_cards.sass deleted file mode 100644 index a1562d0dde1d..000000000000 --- a/website/www/site/assets/scss/_cards.sass +++ /dev/null @@ -1,69 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -.cards - background-image: url(../images/cards_bg.svg) - background-size: cover - background-repeat: no-repeat - background-position: top - text-align: center - margin-bottom: $pad*2 - - .cards__title - +type-h2 - color: #fff - padding-top: $pad-md - margin-bottom: $pad - - .cards__body - max-width: 550px - +type-body - margin: 0 auto - - .cards__cards - margin-bottom: $pad*2 - +md - display: flex - justify-content: center - align-items: center - - .cards__cards__card - background: #fff - box-shadow: $box-shadow - max-width: 300px - margin: 0 auto $pad - padding: $pad*1.5 - +md - margin: 0 $pad/2 - - .cards__cards__card__body - margin-bottom: $pad - +type-h3 - - .cards__cards__card__user - display: flex - justify-content: center - align-items: center - - .cards__cards__card__user__icon - border-radius: 100% - background: #efefef - width: 40px - height: 40px - - .cards__cards__card__user__name - margin-left: $pad/2 diff --git a/website/www/site/assets/scss/_global.sass b/website/www/site/assets/scss/_global.sass index a4dd35517e26..5d8367e45a7c 100644 --- a/website/www/site/assets/scss/_global.sass +++ b/website/www/site/assets/scss/_global.sass @@ -26,13 +26,14 @@ body .body background: #fff margin: 0 auto - padding-top: 130px &:not(.body--index) .body__contained + @media (max-width: $ak-breakpoint-lg) + padding: 0 padding: 0 30px max-width: 1280px - + figure img width: 100% @@ -66,5 +67,13 @@ body width: 100% .container-main-content - padding: 0 20px - position: relative \ No newline at end of file + @media (max-width: $ak-breakpoint-lg) + padding: 0 24px + + padding: 0 22px + position: relative + background-color: #fff + margin-top: 85px + + @media (min-width: $tablet) + margin-top: 0 diff --git a/website/www/site/assets/scss/_graphic.sass b/website/www/site/assets/scss/_graphic.sass index f01c72a12528..cd171f7db00b 100644 --- a/website/www/site/assets/scss/_graphic.sass +++ b/website/www/site/assets/scss/_graphic.sass @@ -16,6 +16,8 @@ */ .graphic + padding: $pad-l $pad + .graphic__image text-align: center line-height: 0 diff --git a/website/www/site/assets/scss/_hero-mobile.scss b/website/www/site/assets/scss/_hero-mobile.scss new file mode 100644 index 000000000000..343c279f4c90 --- /dev/null +++ b/website/www/site/assets/scss/_hero-mobile.scss @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.hero-mobile { + position: relative; + margin-bottom: 0; + display: none; + + .hero-content { + position: absolute; + z-index: 1; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + text-align: center; + width: 100%; + max-width: 506px; + + h3 { + @extend .hero-title; + + text-transform: uppercase; + margin: 0 auto 16px auto; + } + + h1 { + @extend .hero-heading; + + width: 300px; + margin: 0 auto 24px auto; + } + + h2 { + @extend .hero-subheading; + + width: 300px; + margin: 0 auto; + } + } +} + +@media (max-width: $mobile) { + .hero-mobile { + display: inherit; + } +} diff --git a/website/www/site/assets/scss/_hero.sass b/website/www/site/assets/scss/_hero.sass deleted file mode 100644 index 63c22b9282a5..000000000000 --- a/website/www/site/assets/scss/_hero.sass +++ /dev/null @@ -1,156 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -.hero-bg - background-image: url(../images/hero_bg_flat.svg) - background-repeat: no-repeat - background-size: cover - background-position: top center - margin-top: -50px - +md - background-size: 100% - padding-bottom: $pad - -.hero - padding-top: $pad-xl - margin-bottom: $pad-md - position: relative - z-index: 1 - +md - padding-top: $pad-sm - margin-bottom: $pad-xl - - .hero__content - position: relative - z-index: 1 - - .hero__image - bottom: 0 - content: '' - left: 0 - line-height: 0 - position: absolute - right: 0 - top: 0 - z-index: 0 - img - position: absolute - bottom: 0 - width: 100% - - .hero__title - +type-h1 - color: #fff - max-width: 500px - margin: 0 auto $pad - text-align: center - +md - margin: 0 0 $pad - text-align: left - - .hero__ctas - text-align: center - margin-bottom: $pad-md - +md - margin-bottom: 0 - text-align: left - - &--first - margin-bottom: $pad - +md - margin-bottom: $pad-sm - - .hero__subtitle - +type-h3 - color: #fff - max-width: 540px - margin: 0 auto $pad - font-weight: $font-weight-semibold - text-align: center - +md - margin: 0 0 $pad-md - text-align: left - - .hero__blog - .hero__blog__title - +type-h4 - font-weight: $font-weight-bold - margin-bottom: $pad - text-align: center - +md - color: #fff - text-align: left - margin-bottom: $pad/2 - - .hero__blog__cards - +md - display: flex - margin: 0 -10px - - .hero__blog__cards__card - background-color: #fff - color: inherit - box-shadow: $box-shadow - padding: 20px - display: block - transition: transform 300ms ease, box-shadow 300ms ease - position: relative - max-width: 300px - margin: 0 auto $pad - +md - margin: 0 10px - - &:before - background-image: url(../images/card_border.svg) - background-position: top - background-repeat: no-repeat - background-size: cover - content: ' ' - display: block - height: 2px - position: absolute - width: 100% - left: 0 - top: 0 - - &:hover - text-decoration: none - transform: translateY(-8px) - box-shadow: $box-shadow-hover - - .hero__blog__cards__card__title - +type-body - margin-bottom: $pad - - .hero__blog__cards__card__date - +type-body-sm - font-weight: $font-weight-semibold - text-transform: uppercase - letter-spacing: 1px - - .hero__cols - +md - display: flex - min-height: 500px - - .hero__cols__col - width: 50% - display: flex - align-items: flex-end - - &:first-child - align-items: center diff --git a/website/www/site/assets/scss/_hero.scss b/website/www/site/assets/scss/_hero.scss new file mode 100644 index 000000000000..e204d1bf015d --- /dev/null +++ b/website/www/site/assets/scss/_hero.scss @@ -0,0 +1,128 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.hero-desktop { + position: relative; + margin-bottom: 0; + width: 100%; + height: 100%; + display: inherit; + margin-top: -30px; + + .hero-content { + position: absolute; + z-index: 1; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + text-align: center; + + h3 { + @extend .hero-title; + + text-transform: uppercase; + margin: 0 auto 24px auto; + } + + h1 { + @extend .hero-heading; + width: 506px; + + margin: 0 auto 36px auto; + } + + h2 { + @extend .hero-subheading; + + width: 344px; + margin: 0 auto 56px auto; + } + + a { + text-decoration: none; + } + + button { + width: 184px; + height: 46px; + padding: 15px 28px 15px 26px; + border-radius: 100px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + display: flex; + align-items: center; + justify-content: center; + margin: 0 auto; + border: none; + outline: none; + + span { + text-transform: uppercase; + font-size: 14px; + font-weight: bold; + letter-spacing: 0.6px; + color: $color-sun; + } + } + + button:hover { + background-color: $color-white; + box-shadow: 0 4px 20px 0 rgba(0, 0, 0, 0.24), + 0 4px 6px 0 rgba(0, 0, 0, 0.24); + } + + button:focus { + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + } + } +} + +@media (max-width: $tablet) { + .hero-desktop { + margin-top: 70px; + + .hero-content { + h3 { + margin: 0 auto 16px auto; + } + + h1 { + width: 300px; + margin: 0 auto 24px auto; + } + + h2 { + width: 300px; + margin: 0 auto; + } + + button { + display: none; + } + } + } +} + +@media (max-width: $mobile) { + .hero-desktop { + display: none; + } +} diff --git a/website/www/site/assets/scss/_keen-slider.scss b/website/www/site/assets/scss/_keen-slider.scss new file mode 100644 index 000000000000..08a4326e0042 --- /dev/null +++ b/website/www/site/assets/scss/_keen-slider.scss @@ -0,0 +1,40 @@ +/** + * keen-slider 5.3.2 + * The HTML touch slider carousel with the most native feeling you will get. + * https://keen-slider.io + * Copyright 2020-2020 Eric Beyer + * License: MIT + * Released on: 2020-11-10 + */ + +/*# sourceMappingURL=keen-slider.min.css.map */ +// This is pulled from "https://cdn.jsdelivr.net/npm/keen-slider@5.3.2/keen-slider.min.css" to serve the consistency +.keen-slider { + display: flex; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + -webkit-touch-callout: none; + -khtml-user-select: none; + touch-action: pan-y; + -webkit-tap-highlight-color: transparent; +} +.keen-slider, +.keen-slider__slide { + overflow: hidden; + position: relative; +} +.keen-slider__slide { + width: 100%; + min-height: 100%; +} +.keen-slider[data-keen-slider-v] { + flex-wrap: wrap; +} +.keen-slider[data-keen-slider-v] .keen-slider__slide { + width: 100%; +} +.keen-slider[data-keen-slider-moves] * { + pointer-events: none; +} diff --git a/website/www/site/assets/scss/_table-wrapper.sass b/website/www/site/assets/scss/_list-with-icons.scss similarity index 60% rename from website/www/site/assets/scss/_table-wrapper.sass rename to website/www/site/assets/scss/_list-with-icons.scss index 299b0019f62c..fc4880efd295 100644 --- a/website/www/site/assets/scss/_table-wrapper.sass +++ b/website/www/site/assets/scss/_list-with-icons.scss @@ -15,10 +15,29 @@ * limitations under the License. */ -.table-wrapper - > table - @extend .table -.table-bordered-wrapper - > table - @extend .table-bordered + .icon-list{ + margin-top: 64px; + max-width: 640px; + .list-item{ + display:flex; + margin-bottom: 31px; + + .list-item-icon{ + width: 44px; + height: 44px; + display: block; + margin-right: 25px; + } + .list-item-header{ + font-size: 22px; + font-weight: normal; + letter-spacing: normal; + line-height: 1.36; + @media (max-width: $mobile){ + font-weight: 500; + } + } + } + } + \ No newline at end of file diff --git a/website/www/site/assets/scss/_navbar-desktop.scss b/website/www/site/assets/scss/_navbar-desktop.scss new file mode 100644 index 000000000000..2a7e2564467d --- /dev/null +++ b/website/www/site/assets/scss/_navbar-desktop.scss @@ -0,0 +1,180 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.navigation-bar-mobile { + display: none; +} + +.navigation-bar-desktop { + display: flex; + height: 96px; + width: 100%; + align-items: center; + justify-content: space-between; + margin-bottom: 30px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + z-index: 10000; // just to make sure that navbar always on top of other elements + + a { + @extend .component-text; + + color: $color-dark-gray; + letter-spacing: 0.2; + margin-right: 56px; + text-decoration: none; + cursor: pointer; + } + + .navbar-logo { + margin-left: 58px; + + img { + width: 88px; + } + } + + .navbar-links { + display: flex; + align-items: center; + justify-content: space-between; + z-index: 10000; + + :last-child { + margin-right: 0; + } + + .navbar-link { + display: inline-block; + position: relative; + margin-bottom: 2px; + } + + .navbar-link::before { + transition: 0.3; + content: ""; + position: absolute; + background-color: $color-sun; + height: 0%; + width: 100%; + bottom: 0px; + border-radius: 5px; + } + + .navbar-link:hover::before { + height: 2px; + } + + .navbar-dropdown-documentation { + list-style-type: none; + + .dropdown-toggle { + margin: 0; + } + + ul { + width: 209px; + left: -25%; + text-align: center; + border: none; + box-shadow: none; + padding-top: 34px; + padding-bottom: 0; + + a { + @extend .component-text; + } + } + } + } + + .navbar-dropdown-apache { + margin-right: 90px; + list-style-type: none; + + .dropdown-toggle { + margin: 0; + } + + ul { + width: 209px; + left: 70%; + transform: translateX(-50%); + text-align: center; + border: none; + box-shadow: none; + padding-top: 35px; + padding-bottom: 0; + + a { + @extend .component-text; + + margin-right: 0 !important; + } + } + + .arrow-icon { + position: absolute; + top: 3px; + right: -30px; + } + } + + .navbar-dropdown-apache:hover { + .arrow-icon { + opacity: 0.84; + } + } + + .navbar-dropdown { + .dropdown-menu > li > a { + &:hover, + &:focus { + text-decoration: none; + color: $color-dropdown-link-hover-text; + background-color: $color-dropdown-link-hover-bg; + } + } + } + + .dropdown:hover .dropdown-menu { + display: block; + margin-top: 0; + } +} + +@media (max-width: $tablet) { + .navigation-bar-desktop { + display: none; + } + + .navigation-bar-mobile { + display: block; + width: 100%; + height: 64px; + padding: 9px 25.4px 9px 13px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + z-index: 10000; + } + + .page-nav { + margin-top: 30px; + } +} \ No newline at end of file diff --git a/website/www/site/assets/scss/_navbar.sass b/website/www/site/assets/scss/_navbar-mobile.sass similarity index 73% rename from website/www/site/assets/scss/_navbar.sass rename to website/www/site/assets/scss/_navbar-mobile.sass index f4b4ea6bcae7..f216aa723f1d 100644 --- a/website/www/site/assets/scss/_navbar.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -16,15 +16,31 @@ */ .navbar - padding: 15px 0 - + padding: 0 !important + :before + display: none + ::after + display: none .navbar-nav > li > a text-transform: uppercase - + .navbar-link + font-size: 16px; + font-weight: normal; + font-stretch: normal; + font-style: normal; + line-height: normal; + letter-spacing: normal .navbar-header - margin-left: $pad + float: none + display: flex + align-items: center + justify-content: space-between + height: 100% .navbar-brand + padding: 0 + display: flex + align-items: center +md margin-right: $pad @@ -40,16 +56,18 @@ color: $color-dark-gray .navbar-toggle - float: left + margin-right: 16px !important .icon-bar - background-color: $color-dark-gray + background-color: $color-sun + height: 3px + width: 20px - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) display: block .navbar-container - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) background-color: $color-white bottom: 0 min-height: 100vh @@ -59,12 +77,13 @@ top: 0 transition: transform 100ms linear width: calc(100% - 32px) + right: 0 .navbar-nav > li width: 100% &.closed - transform: translateX(-100%) + transform: translateX(100%) &.open transform: translateX(0) @@ -78,7 +97,7 @@ top: 0 transition: opacity 200ms - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) display: block &.closed @@ -89,6 +108,6 @@ opacity: 0.5 width: 100% - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) .navbar-right margin-right: -15px diff --git a/website/www/site/assets/scss/_quotes.scss b/website/www/site/assets/scss/_quotes.scss new file mode 100644 index 000000000000..6d59798289f0 --- /dev/null +++ b/website/www/site/assets/scss/_quotes.scss @@ -0,0 +1,151 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.quotes { + padding: $pad-l $pad; + background-color: $color-medium-gray; + + .quotes-title { + @extend .component-title; + + text-align: center; + border: none; + } + + .quotes-desktop { + display: flex; + justify-content: center; + + .quote-card { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + width: 100%; + max-width: 381px; + height: 474px; + margin: 86px 36px 0 0; + padding: 55px 20px 24px 20px; + border-radius: 16px; + background-color: $color-white; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + margin-right: 36px; + + .quote-text { + @extend .component-quote; + + margin: 108px 0 20px 0; + } + + img { + width: 172px; + } + } + + :last-child { + margin-right: 0; + } + } + + // Sliding feature is only displayed on mobile version + .keen-slider { + display: none; + } + + .dots { + display: none; + } + + .keen-slider { + width: 327px; + margin: 0 auto; + border-radius: 16px; + background-color: $color-white; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + + .keen-slider__slide { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + width: 100%; + max-width: 327px; + height: 468px; + padding: 55px 24px 24px 24px; + + .quote-text { + @extend .component-quote; + + margin: 108px 0 20px 0; + } + + img { + width: 172px; + } + } + } + + .dots { + display: none; + padding: 10px 0; + justify-content: center; + margin-top: 46px; + } + + .dot { + border: none; + width: 13px; + height: 13px; + background: $color-smoke; + border-radius: 50%; + margin: 0 5px; + padding: 4px; + cursor: pointer; + } + + .dot:focus { + outline: none; + } + + .dot--active { + background: $color-sun; + } +} + +@media (max-width: $tablet) { + .quotes { + .quotes-title { + margin-bottom: 64px; + } + + .quotes-desktop { + display: none; + } + + .keen-slider { + display: flex; + } + + .dots { + display: flex; + } + } +} diff --git a/website/www/site/assets/scss/_section-nav.sass b/website/www/site/assets/scss/_section-nav.sass index 61aff9f462ea..0712b6acec56 100644 --- a/website/www/site/assets/scss/_section-nav.sass +++ b/website/www/site/assets/scss/_section-nav.sass @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - .section-nav max-width: 250px position: fixed @@ -97,7 +96,7 @@ @media (max-width: $ak-breakpoint-lg) background-color: $color-light-gray bottom: 0 - left: 0 + right: 0 max-width: 256px position: fixed top: 0 @@ -110,10 +109,7 @@ overflow-y: auto &.closed - transform: translateX(-100%) + transform: translateX(100%) &.open transform: translateX(0) - - - diff --git a/website/www/site/assets/scss/_table-wrapper.scss b/website/www/site/assets/scss/_table-wrapper.scss new file mode 100644 index 000000000000..226c331e1fc6 --- /dev/null +++ b/website/www/site/assets/scss/_table-wrapper.scss @@ -0,0 +1,103 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + $padding:24px; +.table-wrapper{ + table{ + @extend .table; + max-width: 853px; + } + th:nth-child(1) { + @media (max-width: $mobile){ + padding-left:0 !important; + width: 115px; + } + padding-left: 67px+$padding !important; + width: 491px; + } + th:nth-child(2) { + @media (max-width: $mobile){ + width: 111px; + } + width: 181px; + } + th{ + padding: 14px 14px 17px 14px !important; + border: none !important; + border-bottom: 1px solid #ff6d05 !important; + letter-spacing: 2px; + line-height: normal !important; + } + td{ + @media (max-width: $mobile){ + padding: $padding/4 !important; + } + padding: $padding !important; + border-bottom: 1px solid rgba(255, 109, 5, 0.24); + div{ + display: flex; + position: relative; + } + img, svg{ + @media (max-width: $mobile){ + display: none; + } + position: absolute; + top:50%; + transform: translateY(-60%); + } + p{ + @media (max-width: $mobile){ + margin:0; + } + margin-left: 67px; + font-size: 14px; + line-height: 1.57; + } + } + td:nth-child(3) { + @media (max-width: $mobile){ + padding: $padding/3 !important; + padding-left: $padding/4 !important; + } + padding: $padding/3 !important; + padding-left: $padding !important; + div{ + flex-direction: column; + } + a{ + text-decoration: underline; + line-height: 2; + letter-spacing: 0.43px; + } + } + td:nth-child(2) { + background-color: rgba(255, 109, 0, 0.04); + padding-left: $padding/2 !important; + padding-right: 0 !important; + line-height: 1.57; + letter-spacing: 0.43px; + } + a{ + color:#f26628; + } +} + +.table-bordered-wrapper{ + table{ + @extend .table-bordered + } +} \ No newline at end of file diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index 99eacfe4ca04..f81b2eed605b 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - @import "media"; + +@import "media"; .component-title { font-size: 36px; @@ -88,6 +88,44 @@ color: $color-smoke; } +.component-quote { + font-size: 20px; + font-weight: normal; + font-stretch: normal; + font-style: italic; + line-height: 1.44; + letter-spacing: 0.43px; + text-align: center; + color: $color-gray; +} + +.hero-title { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.88; + letter-spacing: 0.8px; + color: $color-white; +} + +.hero-heading { + font-size: 46px; + font-weight: 500; + font-style: normal; + line-height: 1; + letter-spacing: normal; + color: $color-white; +} + +.hero-subheading { + font-size: 20px; + font-weight: normal; + font-style: normal; + line-height: 1.44; + letter-spacing: normal; + color: $color-white; +} + @media (max-width: $tablet) { .component-title { font-size: 28px; @@ -95,4 +133,34 @@ .component-large-header { font-size: 24px; } -} \ No newline at end of file +} + +h1{ + @media (max-width: $tablet){ + font-size: 32px; + line-height: 1.19; + font-weight: 500; + } + font-size: 46px; + font-weight: normal; + line-height: 1; +} +h2{ + @media (max-width: $tablet){ + font-size: 24px; + line-height: 1.25; + font-weight: 500; + } + font-size: 29px; + line-height: 1.24; +} +p{ + font-size: 16px; + font-weight: normal; + line-height: 1.63; + letter-spacing: 0.43px; +} + .hero-heading { + font-size: 32px; + } + diff --git a/website/www/site/assets/scss/_vars.sass b/website/www/site/assets/scss/_vars.sass index df4276a042b1..626313d993b2 100644 --- a/website/www/site/assets/scss/_vars.sass +++ b/website/www/site/assets/scss/_vars.sass @@ -24,6 +24,10 @@ $color-gray: #333333 $color-smoke: #8C8B8E $color-sun: #F26628 $color-silver: #C4C4C4 +$color-medium-gray: #FBFBFB + +$color-dropdown-link-hover-text: #E65D21 +$color-dropdown-link-hover-bg: #FFEDE5 $pad-sm: 15px $pad-s: 24px diff --git a/website/www/site/assets/scss/bootstrap/_navbar.scss b/website/www/site/assets/scss/bootstrap/_navbar.scss index 11e5c01c1585..ee2bb17a847b 100755 --- a/website/www/site/assets/scss/bootstrap/_navbar.scss +++ b/website/www/site/assets/scss/bootstrap/_navbar.scss @@ -210,7 +210,7 @@ // Bars .icon-bar { display: block; - width: 22px; + width: 20px; height: 2px; border-radius: 1px; } diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index e4075a1fe51a..0e406aaa74e8 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -20,26 +20,29 @@ // Globals. @import "_vars.sass"; +@import "_media.scss"; @import "_breakpoints.sass"; @import "_type.sass"; @import "_global.sass"; -@import "_navbar.sass"; +@import "_navbar-mobile.sass"; @import "_typography.scss"; -@import "_media.scss"; // Components. @import "_button.sass"; // Modules. -@import "_cards.sass"; @import "_ctas.sass"; @import "_footer.sass"; @import "_graphic.sass"; @import "_header.sass"; -@import "_hero.sass"; +@import "_hero.scss"; +@import "_hero-mobile.scss"; @import "_logos.scss"; @import "_pillars.scss"; @import "_section-nav.sass"; @import "_page-nav.sass"; -@import "_table-wrapper.sass"; -@import "_calendar.scss"; \ No newline at end of file +@import "_table-wrapper.scss"; +@import "_calendar.scss"; +@import "list-with-icons.scss"; +@import "_quotes.scss"; +@import "navbar-desktop.scss"; diff --git a/website/www/site/content/en/community/_index.md b/website/www/site/content/en/community/_index.md new file mode 100644 index 000000000000..11e6bb4848eb --- /dev/null +++ b/website/www/site/content/en/community/_index.md @@ -0,0 +1,28 @@ +--- +title: "Community Beam" +aliases: + - /use/issue-tracking/ + - /use/mailing-lists/ + - /get-started/support/ +--- + + +# Welcome to the Beam Community! +
+ +Beam is a tool created by community for community. We tirelessly work to make it better and you can do it too! +If you are a data or software developer this is a place for you. + +{{< list_with_icons >}} \ No newline at end of file diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index cca25cda1224..7da187e6526b 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -20,28 +20,34 @@ See the License for the specific language governing permissions and limitations under the License. --> -# Contact Us + +# Contact us! + +
There are many ways to reach the Beam user and developer communities - use whichever one seems best. - -
+
+
-| How to contact us | When to use it | -| ----------------- | ---------------| -| [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list | User support and questions ([Subscribe](mailto:user-subscribe@beam.apache.org)[^1], [Unsubscribe](mailto:user-unsubscribe@beam.apache.org)[^1], [Archives](https://lists.apache.org/list.html?user@beam.apache.org)) | -| [dev@](https://lists.apache.org/list.html?dev@beam.apache.org) mailing list | Development discussions ([Subscribe](mailto:dev-subscribe@beam.apache.org)[^1], [Unsubscribe](mailto:dev-unsubscribe@beam.apache.org)[^1], [Archives](https://lists.apache.org/list.html?dev@beam.apache.org)) | -| [commits@](https://lists.apache.org/list.html?commits@beam.apache.org) mailing list | Firehose of commits, bugs, pull requests, etc. ([Subscribe](mailto:commits-subscribe@beam.apache.org)[^1], [Unsubscribe](mailto:commits-unsubscribe@beam.apache.org)[^1], [Archives](https://lists.apache.org/list.html?commits@beam.apache.org)) | -| [builds@](https://lists.apache.org/list.html?builds@beam.apache.org) mailing list | Firehose of build notifications from Jenkins ([Subscribe](mailto:builds-subscribe@beam.apache.org)[^1], [Unsubscribe](mailto:builds-unsubscribe@beam.apache.org)[^1], [Archives](https://lists.apache.org/list.html?builds@beam.apache.org)) | -| [JIRA bug tracker](https://issues.apache.org/jira/browse/BEAM) | Report bugs / discover known issues | -| [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam) | Ask and answer user support questions | -| [Slack](https://s.apache.org/beam-slack-channel) | Chat with users and developers in the ASF Slack. Note: Please [join the #beam channel](https://s.apache.org/beam-slack-channel) after you [created an account](https://s.apache.org/slack-invite). Please do not ask Beam questions in #general. | +## How to use mailing lists?/What is it? +Here should be a short description on what mailing lists are and how to use them. Also information on how does subscribe, unsubscribe and archives do work (blank email etc). -
+
-If you have questions about how to use Apache Beam, we recommend you try out the [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list, and [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam). -[^1]: To subscribe or unsubscribe, a blank email is fine. +## Available points of contact +Choose the mailing list that suits you best and send us an email! + +
+
+ +{{< table_with_icons >}} +
+ + +If you have questions about how to use Apache Beam, we recommend you try out the [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list, and [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam). If you wish to report a security vulnerability, please contact [security@apache.org](mailto:security@apache.org). Apache Beam follows the typical [Apache vulnerability handling process](https://apache.org/security/committers.html#vulnerability-handling). + diff --git a/website/www/site/content/en/community/powered-by.md b/website/www/site/content/en/community/powered-by.md index ad22039388b9..dd45ece15b4b 100644 --- a/website/www/site/content/en/community/powered-by.md +++ b/website/www/site/content/en/community/powered-by.md @@ -22,4 +22,6 @@ To add yourself to the list, please open a [pull request](https://github.com/apa Apache Beam pipelines within the Google Cloud Platform ecosystem. * **[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx):** TensorFlow Extended (TFX) is an end-to-end platform for deploying production ML pipelines. +* **[Apache Hop (incubating)](http://hop.apache.org):** Hop provides a complete data orchestration (ETL / DI) toolset with visual pipeline development. It supports execution on the main Apache Beam runners. + diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index 417565bba1c3..4a095b70ba38 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -65,7 +65,7 @@ To contribute code, you need - [Docker](https://www.docker.com/) installed for some tasks including building worker containers and testing this website changes locally - [Go](https://golang.org) 1.12 or later installed for Go SDK development - - Python 2.7, 3.5, 3.6, and 3.7. Yes, you need all four versions installed. + - Python 3.6, 3.7, and 3.8. Yes, you need all three versions installed. - pip, setuptools, virtualenv, and tox installed for Python development - for large contributions, a signed [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache @@ -83,6 +83,11 @@ sudo apt-get install \ docker-ce ``` +On some systems (like Ubuntu 20.04) these need to be installed also +``` +pip3 install grpcio-tools mypy-protobuf +``` + You also need to [install Go](https://golang.org/doc/install). Once Go is installed, install goavro: @@ -94,6 +99,15 @@ $ go get github.com/linkedin/goavro gLinux users should configure their machines for sudoless Docker. +Alternatively, you can use the Docker based local development environment to wrap your clone of the Beam repo +into a container meeting the requirements above. + +You can start this container using the [start-build-env.sh](https://github.com/apache/beam/blob/master/start-build-env.sh) +script which is part of the Beam repo: +``` +./start-build-env.sh +``` + ### Connect With the Beam community 1. Consider subscribing to the [dev@ mailing list](/community/contact-us/), especially diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index 38ba4fab7ec9..2182b4b8f438 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -67,6 +67,7 @@ tableElement: columnName fieldType [ NOT NULL ] [Identifier](/documentation/dsls/sql/calcite/lexical/#identifiers) with one of the following values: * `bigquery` + * `bigtable` * `pubsub` * `kafka` * `text` @@ -204,6 +205,137 @@ TYPE bigquery LOCATION 'testing-integration:apache.users' ``` +## Cloud Bigtable + +### Syntax + +``` +CREATE EXTERNAL TABLE [ IF NOT EXISTS ] tableName ( + key VARCHAR NOT NULL, + family ROW + [, family ROW< qualifier cells [, qualifier cells ]* > ]* +) +TYPE bigtable +LOCATION 'googleapis.com/bigtable/projects/[PROJECT_ID]/instances/[INSTANCE_ID]/tables/[TABLE]' +``` + +* `key`: key of the Bigtable row +* `family`: name of the column family +* `qualifier`: the column qualifier +* `cells`: Either of each value: + * `TYPE` + * `ARRAY` +* `LOCATION`: + * `PROJECT_ID`: ID of the Google Cloud Project. + * `INSTANCE_ID`: Bigtable instance ID. + * `TABLE`: Bigtable Table ID. +* `TYPE`: `SIMPLE_TYPE` or `CELL_ROW` +* `CELL_ROW`: `ROW [NOT NULL]]` +* `SIMPLE_TYPE`: on of the following: + * `BINARY` + * `VARCHAR` + * `BIGINT` + * `INTEGER` + * `SMALLINT` + * `TINYINT` + * `DOUBLE` + * `FLOAT` + * `BOOLEAN` + * `TIMESTAMP` + +An alternative syntax with a flat schema: +``` +CREATE EXTERNAL TABLE [ IF NOT EXISTS ] tableName ( + key VARCHAR NOT NULL, + qualifier SIMPLE_TYPE + [, qualifier SIMPLE_TYPE ]* +) +TYPE bigtable +LOCATION 'googleapis.com/bigtable/projects/[PROJECT_ID]/instances/[INSTANCE_ID]/tables/[TABLE]' +TBLPROPERTIES '{ + "columnsMapping": "family:qualifier[,family:qualifier]*" +}' +``` + +* `key`: key of the Bigtable row +* `family`: name of the column family +* `qualifier`: the column qualifier +* `LOCATION`: + * `PROJECT_ID`: ID of the Google Cloud Project. + * `INSTANCE_ID`: Bigtable instance ID. + * `TABLE`: Bigtable Table ID. +* `TBLPROPERTIES`: JSON object containing columnsMapping key with comma-separated + key-value pairs separated by a colon +* `SIMPLE_TYPE`: the same as in the previous syntax + +### Read Mode + +Beam SQL supports reading rows with mandatory `key` field, at least one `family` +with at least one `qualifier`. Cells are represented as simple types (`SIMPLE_TYPE`) or +ROW type with a mandatory `val` field, optional `timestampMicros` and optional `labels`. Both +read the latest cell in the column. Cells specified as Arrays of simple types +(`ARRAY`) allow to read all the column's values. + +For flat schema only `SIMPLE_TYPE` values are allowed. Every field except for `key` must correspond +to the key-values pairs specified in `columnsMapping`. + +Not all existing column families and qualifiers have to be provided to the schema. + +### Write Mode + +Supported for flat schema only. + +### Example + +``` +CREATE EXTERNAL TABLE beamTable( + key VARCHAR NOT NULL, + beamFamily ROW< + boolLatest BOOLEAN NOT NULL, + longLatestWithTs ROW< + val BIGINT NOT NULL, + timestampMicros BIGINT NOT NULL + > NOT NULL, + allStrings ARRAY NOT NULL, + doubleLatestWithTsAndLabels ROW< + val DOUBLE NOT NULL, + timestampMicros BIGINT NOT NULL, + labels ARRAY NOT NULL + > NOT NULL, + binaryLatestWithLabels ROW< + val BINARY NOT NULL, + labels ARRAY NOT NULL + > NOT NULL + > NOT NULL + ) +TYPE bigtable +LOCATION 'googleapis.com/bigtable/projects/beam/instances/beamInstance/tables/beamTable' +``` + +Flat schema example: + +``` +CREATE EXTERNAL TABLE flatTable( + key VARCHAR NOT NULL, + boolColumn BOOLEAN NOT NULL, + longColumn BIGINT NOT NULL, + stringColumn VARCHAR NOT NULL, + doubleColumn DOUBLE NOT NULL, + binaryColumn BINARY NOT NULL +) +TYPE bigtable +LOCATION 'googleapis.com/bigtable/projects/beam/instances/beamInstance/tables/flatTable' +TBLPROPERTIES '{ + "columnsMapping": "f:boolColumn,f:longColumn,f:stringColumn,f2:doubleColumn,f2:binaryColumn" +}' +``` + +Write example: +``` +INSERT INTO writeTable(key, boolColumn, longColumn, stringColumn, doubleColumn) + VALUES ('key', TRUE, 10, 'stringValue', 5.5) +``` + ## Pub/Sub ### Syntax diff --git a/website/www/site/content/en/documentation/io/developing-io-java.md b/website/www/site/content/en/documentation/io/developing-io-java.md index 7de2024cf3c7..7836a3cd06ac 100644 --- a/website/www/site/content/en/documentation/io/developing-io-java.md +++ b/website/www/site/content/en/documentation/io/developing-io-java.md @@ -17,6 +17,9 @@ limitations under the License. --> # Developing I/O connectors for Java +**IMPORTANT:** Use ``Splittable DoFn`` to develop your new I/O. For more details, read the +[new I/O connector overview](/documentation/io/developing-io-overview/). + To connect to a data store that isn’t supported by Beam’s existing I/O connectors, you must create a custom I/O connector that usually consist of a source and a sink. All Beam sources and sinks are composite transforms; however, diff --git a/website/www/site/content/en/documentation/io/developing-io-overview.md b/website/www/site/content/en/documentation/io/developing-io-overview.md index 0ea507f2bf79..c8e0482aa9d3 100644 --- a/website/www/site/content/en/documentation/io/developing-io-overview.md +++ b/website/www/site/content/en/documentation/io/developing-io-overview.md @@ -46,33 +46,32 @@ are the recommended steps to get started: For **bounded (batch) sources**, there are currently two options for creating a Beam source: +1. Use `Splittable DoFn`. + 1. Use `ParDo` and `GroupByKey`. -1. Use the `Source` interface and extend the `BoundedSource` abstract subclass. -`ParDo` is the recommended option, as implementing a `Source` can be tricky. See -[When to use the Source interface](#when-to-use-source) for a list of some use -cases where you might want to use a `Source` (such as -[dynamic work rebalancing](/blog/2016/05/18/splitAtFraction-method.html)). +`Splittable DoFn` is the recommended option, as it's the most recent source framework for both +bounded and unbounded sources. This is meant to replace the `Source` APIs( +[BoundedSource](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html) and +[UnboundedSource](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html)) +in the new system. Read +[Splittable DoFn Programming Guide](/learn/programming-guide/#splittable-dofns) for how to write one +Splittable DoFn. For more information, see the +[roadmap for multi-SDK connector efforts](/roadmap/connectors-multi-sdk/). -(Java only) For **unbounded (streaming) sources**, you must use the `Source` -interface and extend the `UnboundedSource` abstract subclass. `UnboundedSource` -supports features that are useful for streaming pipelines, such as -checkpointing. +For Java and Python **unbounded (streaming) sources**, you must use the `Splittable DoFn`, which +supports features that are useful for streaming pipelines, including checkpointing, controlling +watermark, and tracking backlog. -Splittable DoFn is a new sources framework that is under development and will -replace the other options for developing bounded and unbounded sources. For more -information, see the -[roadmap for multi-SDK connector efforts](/roadmap/connectors-multi-sdk/). -### When to use the Source interface {#when-to-use-source} +### When to use the Splittable DoFn interface {#when-to-use-splittable-dofn} -If you are not sure whether to use `Source`, feel free to email the [Beam dev -mailing list](/get-started/support) and we can discuss the -specific pros and cons of your case. +If you are not sure whether to use `Splittable DoFn`, feel free to email the +[Beam dev mailing list](/get-started/support) and we can discuss the specific pros and cons of your +case. -In some cases, implementing a `Source` might be necessary or result in better -performance: +In some cases, implementing a `Splittable DoFn` might be necessary or result in better performance: * **Unbounded sources:** `ParDo` does not work for reading from unbounded sources. `ParDo` does not support checkpointing or mechanisms like de-duping @@ -90,22 +89,40 @@ performance: jobs. Depending on your data source, dynamic work rebalancing might not be possible. -* **Splitting into parts of particular size recommended by the runner:** `ParDo` - does not receive `desired_bundle_size` as a hint from runners when performing - initial splitting. +* **Splitting initially to increase parallelism:** `ParDo` + does not have the ability to perform initial splitting. For example, if you'd like to read from a new file format that contains many records per file, or if you'd like to read from a key-value store that supports read operations in sorted key order. -### Source lifecycle {#source} -Here is a sequence diagram that shows the lifecycle of the Source during - the execution of the Read transform of an IO. The comments give useful - information to IO developers such as the constraints that - apply to the objects or particular cases such as streaming mode. - - -![This is a sequence diagram that shows the lifecycle of the Source](/images/source-sequence-diagram.svg) +### I/O examples using SDFs +**Java Examples** + +* [Kafka](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java#L118): +An I/O connector for [Apache Kafka](https://kafka.apache.org/) +(an open-source distributed event streaming platform). +* [Watch](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java#L787): +Uses a polling function producing a growing set of outputs for each input until a per-input +termination condition is met. +* [Parquet](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java#L365): +An I/O connector for [Apache Parquet](https://parquet.apache.org/) +(an open-source columnar storage format). +* [HL7v2](https://github.com/apache/beam/blob/6fdde4f4eab72b49b10a8bb1cb3be263c5c416b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java#L493): +An I/O connector for HL7v2 messages (a clinical messaging format that provides data about events +that occur inside an organization) part of +[Google’s Cloud Healthcare API](https://cloud.google.com/healthcare). +* [BoundedSource wrapper](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L248): +A wrapper which converts an existing [BoundedSource](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/BoundedSource.html) +implementation to a splittable DoFn. +* [UnboundedSource wrapper](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java#L432): +A wrapper which converts an existing [UnboundedSource](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/UnboundedSource.html) +implementation to a splittable DoFn. + +**Python Examples** +* [BoundedSourceWrapper](https://github.com/apache/beam/blob/571338b0cc96e2e80f23620fe86de5c92dffaccc/sdks/python/apache_beam/io/iobase.py#L1375): +A wrapper which converts an existing [BoundedSource](https://beam.apache.org/releases/pydoc/current/apache_beam.io.iobase.html#apache_beam.io.iobase.BoundedSource) +implementation to a splittable DoFn. ### Using ParDo and GroupByKey @@ -157,7 +174,6 @@ example: cannot be parallelized. In this case, the `ParDo` would open the file and read in sequence, producing a `PCollection` of records from the file. - ## Sinks To create a Beam sink, we recommend that you use a `ParDo` that writes the @@ -169,8 +185,6 @@ For **file-based sinks**, you can use the `FileBasedSink` abstraction that is provided by both the Java and Python SDKs. See our language specific implementation guides for more details: -* [Developing I/O connectors for Java](/documentation/io/developing-io-java/) -* [Developing I/O connectors for Python](/documentation/io/developing-io-python/) diff --git a/website/www/site/content/en/documentation/io/developing-io-python.md b/website/www/site/content/en/documentation/io/developing-io-python.md index 039b633c4209..7c7705bb1be8 100644 --- a/website/www/site/content/en/documentation/io/developing-io-python.md +++ b/website/www/site/content/en/documentation/io/developing-io-python.md @@ -19,6 +19,9 @@ limitations under the License. --> # Developing I/O connectors for Python +**IMPORTANT:** Please use ``Splittable DoFn`` to develop your new I/O. For more details, please read +the [new I/O connector overview](/documentation/io/developing-io-overview/). + To connect to a data store that isn’t supported by Beam’s existing I/O connectors, you must create a custom I/O connector that usually consist of a source and a sink. All Beam sources and sinks are composite transforms; however, diff --git a/website/www/site/content/en/documentation/programming-guide.md b/website/www/site/content/en/documentation/programming-guide.md index 3eee8fef6d16..477152c23cb1 100644 --- a/website/www/site/content/en/documentation/programming-guide.md +++ b/website/www/site/content/en/documentation/programming-guide.md @@ -31,7 +31,7 @@ how to implement Beam concepts in your pipelines. {{< language-switcher java py >}} {{< paragraph class="language-py" >}} -The Python SDK supports Python 2.7, 3.5, 3.6, and 3.7. New Python SDK releases will stop supporting Python 2.7 in 2020 ([BEAM-8371](https://issues.apache.org/jira/browse/BEAM-8371)). For best results, use Beam with Python 3. +The Python SDK supports Python 3.6, 3.7, and 3.8. Beam 2.24.0 was the last Python SDK release to support Python 2 and 3.5. {{< /paragraph >}} ## 1. Overview {#overview} diff --git a/website/www/site/content/en/documentation/sdks/python-dependencies.md b/website/www/site/content/en/documentation/sdks/python-dependencies.md index 80130e5fb1cd..4e0bf0a36c51 100644 --- a/website/www/site/content/en/documentation/sdks/python-dependencies.md +++ b/website/www/site/content/en/documentation/sdks/python-dependencies.md @@ -41,18 +41,10 @@ Dependencies for your Beam SDK version are listed in `setup.py` in the Beam repo You can also retrieve the dependency list from the command line using the following process: -1. Create a clean virtual environment on your local machine. - - Python 3: - - ``` - $ python3 -m venv env && source env/bin/activate - ``` - - Python 2: +1. Create a clean virtual environment on your local machine using a supported python version. ``` - $ pip install virtualenv && virtualenv env && source env/bin/activate + $ python -m venv env && source env/bin/activate ``` 2. [Install the Beam Python SDK](/get-started/quickstart-py/#download-and-install). diff --git a/website/www/site/content/en/get-started/quickstart-py.md b/website/www/site/content/en/get-started/quickstart-py.md index b2a503e5803d..5036ee0020b7 100644 --- a/website/www/site/content/en/get-started/quickstart-py.md +++ b/website/www/site/content/en/get-started/quickstart-py.md @@ -23,13 +23,13 @@ If you're interested in contributing to the Apache Beam Python codebase, see the {{< toc >}} -The Python SDK supports Python 2.7, 3.5, 3.6, and 3.7. New Python SDK releases will stop supporting Python 2.7 in 2020 ([BEAM-8371](https://issues.apache.org/jira/browse/BEAM-8371)). For best results, use Beam with Python 3. +The Python SDK supports Python 3.6, 3.7, and 3.8. Beam 2.24.0 was the last release with support for Python 2.7 and 3.5. ## Set up your environment ### Check your Python version -The Beam SDK requires Python 2 users to use Python 2.7 and Python 3 users to use Python 3.5 or higher. Check your version by running: +The Beam SDK requires Python users to use Python version 3.6 or higher. Check your version by running: {{< highlight >}} python --version diff --git a/website/www/site/content/en/get-started/wordcount-example.md b/website/www/site/content/en/get-started/wordcount-example.md index 50634c75b9ad..940eb133bacf 100644 --- a/website/www/site/content/en/get-started/wordcount-example.md +++ b/website/www/site/content/en/get-started/wordcount-example.md @@ -147,7 +147,7 @@ Pipeline p = Pipeline.create(options); {{< /highlight >}} {{< highlight go >}} -p := beam.NewPipeline +p := beam.NewPipeline() s := p.Root() {{< /highlight >}} diff --git a/website/www/site/data/en/community_list.yaml b/website/www/site/data/en/community_list.yaml new file mode 100644 index 000000000000..2f667b944b63 --- /dev/null +++ b/website/www/site/data/en/community_list.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- title: Join Beam Community + body: Here you will find real life cases. + icon: icons/community/bee-icon.svg +- title: Contact us! + body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of them info all of the info on who to contact. + icon: icons/community/envelope-icon.svg +- title: Keep up with Beam + body: Check out our social media and be first to know about any updates. + icon: icons/community/message-icon.svg +- title: Meet up with us! + body: Want to meet us in person? (or monitor-person). This is the place to go. + icon: icons/community/calendar-icon.svg +- title: Promotion + body: Need some marketing materials? Here you will find it all. + icon: icons/community/box-icon.svg diff --git a/website/www/site/data/en/contact_us.yaml b/website/www/site/data/en/contact_us.yaml new file mode 100644 index 000000000000..17af7beceac1 --- /dev/null +++ b/website/www/site/data/en/contact_us.yaml @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- purpose: User support and questions + contact: { email: user@, link: "https://lists.apache.org/list.html?user@beam.apache.org", add: mailing list } + icon: icons/community/contact-us/question-mark.svg + actions: [{action: Subscribe,link: "mailto:user-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:user-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?user@beam.apache.org"}] +- purpose: Development discussions + contact: { email: dev@, link: "https://lists.apache.org/list.html?dev@beam.apache.org", add: mailing list} + icon: icons/community/contact-us/discussion.svg + actions: [{action: Subscribe,link: "mailto:dev-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:dev-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?dev@beam.apache.org"}] +- purpose: Firehose of commits, bugs, pull requests, etc. + contact: { email: commits@, link: "https://lists.apache.org/list.html?commits@beam.apache.org", add: mailing list } + icon: icons/community/contact-us/gitArrows.svg + actions: [{action: Subscribe,link: "mailto:commits-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:commits-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?commits@beam.apache.org"}] +- purpose: Firehose of build notifications from Jenkins + contact: { email: builds@, link: "https://lists.apache.org/list.html?builds@beam.apache.org", add: mailing list } + icon: icons/community/contact-us/notification.svg + actions: [{action: Subscribe,link: "mailto:builds-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:builds-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?builds@beam.apache.org"}] +- purpose: Report bugs / discover known issues + contact: { email: JIRA bug tracker, link: "https://issues.apache.org/jira/browse/BEAM" } + icon: icons/community/contact-us/bug.svg + + actions: [{action: '‐'}] +- purpose: Ask and answer user support questions + contact: { email: StackOverflow, link: "https://stackoverflow.com/questions/tagged/apache-beam" } + icon: icons/community/contact-us/messages.svg + actions: [{action: '‐'}] +- purpose: 'Chat with users and developers in the ASF Slack. Note: Please join the #beam channel after you created an account. Please do not ask Beam questions in #general.' + contact: { email: Slack, link: "https://s.apache.org/beam-slack-channel" } + icon: icons/community/contact-us/knot.svg + actions: [{action: '‐'}] diff --git a/website/www/site/data/en/quotes.yaml b/website/www/site/data/en/quotes.yaml new file mode 100644 index 000000000000..537171a7d4c2 --- /dev/null +++ b/website/www/site/data/en/quotes.yaml @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: +# Placeholder texts should be updated later +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png \ No newline at end of file diff --git a/website/www/site/i18n/home/en.yaml b/website/www/site/i18n/home/en.yaml index 8a1ac53eecf8..6ae87c04f465 100644 --- a/website/www/site/i18n/home/en.yaml +++ b/website/www/site/i18n/home/en.yaml @@ -10,10 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- id: home-hero-title - translation: "Apache Beam: An advanced unified programming model" -- id: home-hero-subtitle - translation: "Implement batch and streaming data processing jobs that run on any execution engine." - id: home-learn-more translation: "Learn more" - id: home-try-beam diff --git a/website/www/site/data/en/cards.yaml b/website/www/site/i18n/home/hero/en.yaml similarity index 60% rename from website/www/site/data/en/cards.yaml rename to website/www/site/i18n/home/hero/en.yaml index a2981e7fba86..85f4728d86be 100644 --- a/website/www/site/data/en/cards.yaml +++ b/website/www/site/i18n/home/hero/en.yaml @@ -10,9 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -- quote: "A framework that delivers the flexibility and advanced functionality our customers need." - name: –Talend -- quote: "Apache Beam has powerful semantics that solve real-world challenges of stream processing." - name: –PayPal -- quote: "Apache Beam represents a principled approach for analyzing data streams." - name: –data Artisans +- id: home-hero-title + translation: Introducing Apache Beam +- id: home-hero-heading + translation: An advanced unified programming model +- id: home-hero-subheading + translation: Implement batch and streaming data processing jobs that run on any execution engine. +- id: home-hero-button + translation: Install Beam diff --git a/website/www/site/i18n/home/quotes/en.yaml b/website/www/site/i18n/home/quotes/en.yaml new file mode 100644 index 000000000000..95c8e85b6453 --- /dev/null +++ b/website/www/site/i18n/home/quotes/en.yaml @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- id: home-quotes-title + translation: "They tried it out" \ No newline at end of file diff --git a/website/www/site/i18n/navbar/en.yaml b/website/www/site/i18n/navbar/en.yaml index fd22c3a73821..ae6255e50683 100644 --- a/website/www/site/i18n/navbar/en.yaml +++ b/website/www/site/i18n/navbar/en.yaml @@ -16,10 +16,12 @@ translation: "Get Started" - id: nav-documentation translation: "Documentation" +- id: nav-documentation-general + translation: "General" - id: nav-languages translation: "Languages" - id: nav-runners - translation: "RUNNERS" + translation: "Runners" - id: nav-roadmap translation: "Roadmap" - id: nav-contribute diff --git a/website/www/site/layouts/_default/baseof.html b/website/www/site/layouts/_default/baseof.html index 6c245c51ac9f..b8b6e26cdfd9 100644 --- a/website/www/site/layouts/_default/baseof.html +++ b/website/www/site/layouts/_default/baseof.html @@ -22,10 +22,12 @@ {{ block "pillars-section" . }}{{ end }} {{ block "graphic-section" . }}{{ end }} {{ block "calendar-section" . }}{{ end }} + {{ block "quotes-section" . }}{{ end }} + {{ block "quotes-mobile-section" . }}{{ end }} {{ block "logos-section" . }}{{ end }} - {{ block "cards-section" . }}{{ end }} {{ block "ctas-section" . }}{{ end }} {{ partial "footer.html" . }} + {{ partial "hooks/body-end.html"}} diff --git a/website/www/site/layouts/community/baseof.html b/website/www/site/layouts/community/baseof.html index f6b0664dd3d4..c84cee0bedc2 100644 --- a/website/www/site/layouts/community/baseof.html +++ b/website/www/site/layouts/community/baseof.html @@ -11,30 +11,30 @@ */}} - - - {{ partial "head.html" . }} - - - {{ partial "header.html" . }} -
-
- - -
- - - -
- {{ .Content }} -
-
- {{ partial "footer.html" . }} - - + + + {{ partial "head.html" . }} + + + {{ partial "header.html" . }} +
+
+ + +
+ + + +
+ {{ .Content }} +
+
+ {{ partial "footer.html" . }} + + \ No newline at end of file diff --git a/website/www/site/layouts/index.html b/website/www/site/layouts/index.html index bbfdf71b9132..4f7f0a536587 100644 --- a/website/www/site/layouts/index.html +++ b/website/www/site/layouts/index.html @@ -11,47 +11,28 @@ */}} {{ define "hero-section" }} -
-
-
-
-
-
- {{ T "home-hero-title" }} -
-
- {{ T "home-hero-subtitle" }} -
-
- {{ T "home-learn-more" }} - {{ T "home-try-beam" }} - {{ T "home-download-beam" . }} -
-
- {{ T "home-java-quickstart" }} - {{ T "home-python-quickstart" }} - {{ T "home-go-quickstart" }} -
-
-
-
-
-
- {{ T "home-hero-blog-title" }} -
-
- - {{ range ( where site.RegularPages "Section" "blog" | first 3 ) }} - -
{{ .Title }}
-
{{ .Date.Format "Jan 2, 2006" }}
- - {{ end }} - -
-
-
-
+
+
+

{{ T "home-hero-title" }}

+

{{ T "home-hero-heading" }}

+

{{ T "home-hero-subheading" }}

+ + + +
+
+
+
+

{{ T "home-hero-title" }}

+

{{ T "home-hero-heading" }}

+

{{ T "home-hero-subheading" }}

{{ end }} @@ -154,6 +135,32 @@

{{ end }} +{{ define "quotes-section" }} +
+
+ {{ T "home-quotes-title" }} +
+ +
+ {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $quote := $data.quotes }} + {{ partial "quotes/quote.html" (dict "icon" $quote.icon "text" $quote.text "logoUrl" $quote.logoUrl) }} + {{ end }} +
+ + {{/* + The id "my-keen-slider" and "dots" should be named as defaults to make the external library (Keen Slider) works well + */}} +
+ {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $quote := $data.quotes }} + {{ partial "quotes/quote-mobile.html" (dict "icon" $quote.icon "text" $quote.text "logoUrl" $quote.logoUrl) }} + {{ end }} +
+
+
+{{ end }} + {{ define "logos-section" }}
@@ -169,38 +176,6 @@

{{ end }} -{{ define "cards-section" }} -
-
-
- {{ T "home-cards-title" }} -
-
- {{ $data := index $.Site.Data .Site.Language.Lang }} - {{ range $card := $data.cards }} -
-
- {{ $card.quote }} -
-
- {{/* TODO: Implement icons (Original comment from Jekyll) -
-
- */}} -
- {{ $card.name }} -
-
-
- {{ end }} -
-
- {{ T "home-cards-body" }} {{ T "home-contribute" }} {{ T "home-section" }}. -
-
-
-{{ end }} - {{ define "ctas-section" }}
diff --git a/website/www/site/layouts/partials/head.html b/website/www/site/layouts/partials/head.html index b3773320be6d..c017268175e8 100644 --- a/website/www/site/layouts/partials/head.html +++ b/website/www/site/layouts/partials/head.html @@ -45,7 +45,7 @@ - + + + + + diff --git a/website/www/site/layouts/partials/quotes/quote-mobile.html b/website/www/site/layouts/partials/quotes/quote-mobile.html new file mode 100644 index 000000000000..756fe3be39cb --- /dev/null +++ b/website/www/site/layouts/partials/quotes/quote-mobile.html @@ -0,0 +1,21 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} + +
+
+ {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +
+
{{ .text | markdownify }}
+ Quote Logo +
\ No newline at end of file diff --git a/website/www/site/layouts/partials/quotes/quote.html b/website/www/site/layouts/partials/quotes/quote.html new file mode 100644 index 000000000000..45f7ccdd34cf --- /dev/null +++ b/website/www/site/layouts/partials/quotes/quote.html @@ -0,0 +1,21 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} + +
+
+ {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +
+
{{ .text | markdownify }}
+ Quote Logo +
\ No newline at end of file diff --git a/website/www/site/layouts/partials/section-menu/en/community.html b/website/www/site/layouts/partials/section-menu/en/community.html index cbefd010143a..2034d12c8b62 100644 --- a/website/www/site/layouts/partials/section-menu/en/community.html +++ b/website/www/site/layouts/partials/section-menu/en/community.html @@ -9,7 +9,6 @@ See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. */}} -
  • Community
  • Integrations
  • Contact Us
    • @@ -31,3 +30,4 @@ width="14" height="14" alt="External link to Beam roster"> + diff --git a/website/www/site/layouts/shortcodes/list_with_icons.html b/website/www/site/layouts/shortcodes/list_with_icons.html new file mode 100644 index 000000000000..c5f3b63a6401 --- /dev/null +++ b/website/www/site/layouts/shortcodes/list_with_icons.html @@ -0,0 +1,29 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. + */}} + + {{ $data := index $.Site.Data .Site.Language.Lang }} +
    + {{ range $item := $data.community_list }} +
    +
    + {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +
    +
    +
    {{ .title | markdownify }}
    +

    {{ .body | markdownify }}

    +
    +
    + + {{ end }} +
    \ No newline at end of file diff --git a/website/www/site/layouts/shortcodes/table_with_icons.html b/website/www/site/layouts/shortcodes/table_with_icons.html new file mode 100644 index 000000000000..4ac8dcaf221f --- /dev/null +++ b/website/www/site/layouts/shortcodes/table_with_icons.html @@ -0,0 +1,51 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} +{{ $data := index $.Site.Data .Site.Language.Lang }} + +
    + + + + + + + {{ range $item := $data.contact_us }} + + + + + + {{ end }} +
    PURPOSECONTACTACTION
    +
    + {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +

    {{.purpose | safeHTML}}

    +
    +     		+ {{.contact.email}} + {{ if .contact.add }} + {{ .contact.add }} + {{end}} + +
    + {{ range .actions }} + {{ if .link }} + {{.action |safeHTML }} + {{ else }} + {{ .action | safeHTML }} + {{end}} + {{ end }} +
    +
    +
    \ No newline at end of file diff --git a/website/www/site/static/images/beam_logo_navbar_mobile.png b/website/www/site/static/images/beam_logo_navbar_mobile.png new file mode 100644 index 0000000000000000000000000000000000000000..bbc0ffe0edc7c0e4ad580dfe1eec876788db4c39 GIT binary patch literal 1497 zcmV;~1t$85P)r3AFqT;zTe&Qm#9nL31Mu6mYIZIcA=-_46qPFTiu)V7Q76wNCrUEnx{R8RB$gW68hsN{g3Ss zSak3OWH86k>`<2GU=>57!)6zG;d#D<60rcB+7u4o*6>dH1SM#TKzQf+{)f4yDS{m(V=on8G4t}o3sFg zjPZ3b#I`U|Tsyy|aqKEu`;LKuaFzwZs&bN`w9RU4&ga@9iNr=~`A$ZHM13tw9UER6 z?%RwIX%5!b2$^Q)q&KoK5!j8uF7XRwQX=* zauGzDrD4xfBp>dF@yL1}Yc8z>SypZs2;ph6>OugdQL{SjQe!jNH>f=yQmYlUELMaU z{JmxqdK2_S7s};*&^t6=>oW?Vq2M0E4*Gezm9QQVYgU7Wi;*T^f)yd}X&$7g6<>4I z(Xs2{Rb8HD?UF-a>6M4mw4rCA=4{sGx@MNO%W0;eQ99Zw_L=jmOutqM%4Si+>~=A) znmRPwuK_C+45B(G-W-H8-?j_LU@Cy-xtdjt59;Om;QbkQeKTQL^A2a!xS$8wN4`h( zuecBD)v?}Guj=wl!gAIAD|@UtUtE_OuEHLhLUq_e^$3sb*_z@=Ma!uH2O7?rmF;0m zwj5SfnD|fGgkw8q!To0rfAQ*Aj++Las=A-8{bu6#u(ewPtE%*lAsgnf4eO)}cS=Q= z6ae!G_xE_Y^u~3l9|f>bScgX`^Elg40*)$vax^TCQhU#q&8@e6mp>k5e8aGhuY=0& zj9b>2r@aUoyhu2sOvnQz$Nu2Q@rhk89Do&dZsY%$0*MY*#2z zuvz6Q%RW}ptv|D+!ijqusAJpGmbSFzDwcl%tBM(%g<%ci00000NkvXXu0mjfa?Z!q literal 0 HcmV?d00001 diff --git a/website/www/site/static/images/quote-paypal-logo.png b/website/www/site/static/images/quote-paypal-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..0bf48f2bfd81a23fc81585f0ab0b53d9d8d20acf GIT binary patch literal 16047 zcmeHOWmj8Wl*Tg0#i$n|`rm!}HqGC;`!(9x6Pu@bR&2b_dAfq(GgK-h@3%t;|E!5maWgsdov zG7@qTQA6+K;gKzVJEsY}ymq$Q6!E0ZY-Ie{eVaQx;M#L3sjOVAw>|r#-;YKef-CX= zH~$x|!SrTo*Z^1k{EgC#%AJ7EjLY2zi3l%74uFCNgtsu@>IlShAaGES@KxzPBYG+Y zjatfpL~S2wow5`qbO5apVr>_7I2k9&Q##}Xo_ZcTsH&Q0;EVPcJ@ewkUR9kS12JOXwE&BTo$A2Z|`a(B2bE{u?hk`;f@_bZ6k3tH3w(9 zzmwqI|86yh8iu{}TH&&W>6ivsdst@UN^oZw1wtOfth+s^Y3EoKyIW~Q-;katB?qws zl{6(Z(WY{^LsW*QtV3R6plw#}ZOHgmD{{6tpuP1g97B(dP$zU9YAH!I8QVQ9%JYE6w_?ZkAFV_!1F$ zR=Dtby1Ig+btYpI)+_)1{SzbNvQQ+X3Bd(?sYc+%{j+a^!%?AkbamXcKn{s^S|g?o zVF=1YK90bnS(MRNno_} zyR8rJ7pz_?H~nf8X+GFb^2{lhlO$Cqy9NLPex2Mq82Qr*2=O*ReXPH>2|k7Wt-4y^ z;>@gL(4OM6U*2EwI;sG?zE*X%52d zSZzxtQ;}%^C;gL!&G&R(JKof}Kt8u)?eYGj4)c*D&Y6HZyCq&Ui`8n=ftzn4H@_qK zcR!~f*;0d0mR;#M5nKr+yfr54}An_FJPh z8YxKkUn|smR^PfCd&cCkc)fA!zgn8E0}%x`a>YOE7u;!URI&ovypRl~-ZvyR4IlvD z)Xk=zXk?;1&a%8uH|})F^HL$MfAwlDqCL;MQSZKNd9l7yS|AehK<$X;iSHHzUQC|F zP@iz7&O1868$GnRb;ciG0Z0`i6NL)NY2g%JR!CU~CZP5!5z-wblAIl=l6C`V!FJxj zyCVAxYQn9JL&(O)#I!UHB7X;=`ehUx^o|*Mh5%~a-mR&tOM(UzvUAfhH{bD^sX-4R zy|i(+S^|ojSbZAw=w12LMu$0W;j>tT1L!e(T1xOiKjF?;s5IF(0bBv zr275)jp?(ujA=Q=X3$(PSwnQwK>t`epVsZ^%2Te`+rvbs=*#uYJm(iRTSM}{KUv>m z{h#y0A6@7U|LNR!+I4h&*Q9a?8}+D?jiqo{1NQ>-6te_XtRe|Lpi{m%;FIr)O#Oe= zAofs5u1ptiUsw?FG7d6b)7|6Ek?*lDH&HrU(|Qm#1Gk;-AN>UxHm;M!`mKV9lZs3I zO&<1Bza~GuW2+#Bfjx&@@yCJOaEeEUQG9O-*ICt*C>cF&MjyWYEVYuGzXLSYQ;OR> zJ-bE=hXKCsZEyOJm4K|HbljTaO`*BOc+Jz`3qhCtbr#*a62z=vTXeOHC~u14*6)J} zp~*m}q1`5wA{BU27XWql(!*ee}9uA$dz1MbVx{GC>%+ldLe_Eca(35>S!u2^d z^|vX1cJG-(uG({-wQ~RKMqjruV*JL!S3!`R&7@Vy4f}NS`{^86`%O{`8}1R+H%$re z=&?JK5Y*O^cF38U;*oi{49|}f9`p}c_A4zM32%te1DVj6uz-uczuX)K zc3n=K7@n>?t7~#cAbce_CY}uDa_5Zo-n%KY#DTdRwWn0q9f{clXQSk^McsPs7_=Nj zoc0BH50scdckC6ri0e5f@bMoUvFAxHj%dAZIt1mozOlzNFcwG^p@9}n3&WG>TuYs?lkwJL(O#GHZ3l~^hA#VXN|XB zvPp`j+iCEkbkJeRp{b6G&zvD;mkXfUVcYjlFp3*R& zb}MrEZ3yMC?^UYes%u-7=acI}pD(KGoI~Y-8nk;&MecWCybE!$!wK{*MQ`0^>DHqt zu$?JVdij-Q)UT6TBz@hbnXD$*;GPi2Cpkj)iwWFUPy4m0TuX@ami_qg2zb5$IAI zuA#S)ej7hdH}V3N#1_~lmC*&|8`n!$Qf)?)@cg`;n5Fypr2LRocG8d-P|vo~*lAw+ z%Ra?`l7Vq;$6Q(Ppg^;>Av3-FVm%nAMs{!`=DOK~V?dpv{IC6n=U%2(;-@zZ3h98k zghlnWFff>hPZ5#ZX$ICH)9?VQ_fa0JTXf&DmQ66Q4UU!Fvj1?dH(?x8^xl30ecODa zZ`!yLgZN`SrFx7{RIeHJd`(6!$|ZPc>O{vZ{tyfLw7nyt#-*TP& zDpPEz{pFe8&y+YN=tT0lQ}`ojX4A{t+bP>D86@8&y#8{i)p@z@pDUa{vl>k`h?@Ci z3o$g2#r-?nDM3flwExzgE1j7Bu|9Ek&H$Zft~(f&)Ws3OnDoR^Pl)F3NmEnPjn5eV zI&BKLe%e}#Zf7L_0il72z#~>Eo%cl6v_I-#HeWXO5Y}fbC!F z3p{pK1d;w18a=7(I?X^5cyly=)eex>v9tB2bfn>5ZoPlE4VpNmd8V zPu7>U*A;c+>1;ENervHc)n%@Dk@Vs93tFZdJ3ffFA|h7wF+wCrbidDy$5=SN$+4Yz zZzS$pqIISIq+%r^Hh+%DyBW$K`-k>67zNG9%*Zx6I;mntU2uezkW=dCEd7#Iem8cE z$)BHE6=)<;wNN!8$QA7#FDXID?9;D!uVZ5=9Kp4-hk8QMYyPhH8{rdqcIbNbBDb@?}Y^`i}yl{$X4G zp;f5R?3uYUR^Hh?PE*|hEHUhzTh39K zXu~l%FCy@>J;%a4-fNP;^9ac8^Z)67^#^?nx3s-yZd{8YqtZ&H+xFDFcSoKkrmMzd zMv2dLf1#FsfSf)?sD3Bdb1*<8J*Rtj?3?oYJ-^rI2iq)<%^$^%Ri;V|F4H|!U-W=# zGg)S&@DqUh*jE{rYK}3-wmXr#<(U|+rZ(_KK+`i8SrEUy=qTfnA_j(&%sNVDNq5@E z3%#ySh_KD9!}qBuPCdhCvNClHar{*zqCGs~PK=UuQx!StK_PFgZ^iU*w)Edz!LL7H z*%%#CB}+2UWM%v9;=AjJ4E1hVWYgp_3ukNTyNbWcI^(v43eb0yx$u>$XgX&{u19mx z06bINCdN$1rn?@eOIlFXcL*O{$G4PGek?HWF!V#h#zGt&My5t!y!ILOq)3N<%1TMw zJwv`;yT2BS2N2kX$k+1Gek2+D8c0lB#x-!A4nh4;W)-MJmM1N+UJO1bR09(QyBKP8 z4f!>tfp`@>kQ%hHnx@I<5G+2(0?2&MySc9&wwC`mGl%o2dSn1d|CvUw{OQO2glkO^ zi{QvJ-G>m@81Lv)hHf|^7B{hlf=$+BbGbeA5B&};%nwFn#Kn3=>^Y2g6#opi(b=gd zD`tzn+})Bqoc31tuU-sLp209u6~33`4u2JW$B{ma6`@)X@nLhEI>2qxvmvjo!E z3m{1)E_nTx)YUWW*s5V52IQ&MIto1Q=POY7UaRgpkG)Jm9z0J|oqK=Sx1R9gEFN{- zpR##W^(tYY62|Ba=Y|m)cx==)IUULNP_4SJ`L=FBEtRaaz75m3XXCK7?B}g)Rh|U} z*H*%6%#ECCuyxBBN4eD!gn#V)o3l+H&08w*-WRVIF>g;Zu%!iELO^RzVET8UU5v4p zecMIC(`Q-U=z>_+|b_Hg`JC^Qh zwuuY@=U_$zL1_xV`;Bpr*ErZtO%d=@-+7J0v>)`3fm&bj8OW`xegq8rsigDu4SDAE zePumowAz=#@Tq6&!G>OeLPk_+)FKHo!R&B#nhH|mVRQ$y`RU6`r;Z~BDzUpB_YB%* z%)>b&*7*?chQ3w6)22MB<7x8!D@}o&g|u{EWYg=?T-$)R>&bNC2>iX6!ziwvf2Rra z($$2RKNGQ9Gg7RTd=i_#yqb*$0U2eC3Wn@NralQ2qQaCr;D$Y139r zGqq$`!L$LWUsD{7+Y5buH>)9lEq4+wHGAi zIec4dPl_8bu=~;wn>BOLv`X8=JWwOU#i!ftxuhn1W+`_^^xT<*uoJ~KexB`fDZhv* zRNS09HXeE$sFcQCTGxSxf`7Zi}N*MtsXf$9-r{r3wMYG^{y&vzm~kr2}dSV zu0~Gyez<*WUh{jJg6G4{mR+=UoK`sVY7q4!3bMD)XJ}+ zlMUy#hIlHVTISi!V#>a|R}qB@nc!x-G3si~GeDBh;XA?j0BD42df||9PY8Nz4~GkB z*3llcU*}3js7kY@-~N_J@CI9kyaR7ZRZHLX{rf->MTQa`wuS{o{@$pk+{u)u*7G3c z^WK8)lu6M3h*C|*AZ2He%AO5M>1ar5CQuFwmG8{xXMq5>HpVqyc`to_4)hpj*=biQNW5j71>13lab_FANsymVNL0jrsg6szxW8Ex-|MQ z;)!)OvHDfa)rKM!!G3F@+!^nIr9s!HFUD`#bXZ*bmFa9N%D9SNr_Fo3+VnWGzg%ke zit!-NnR_f@MuzFz>v>;z<_ePWMvPBWP8N)~Z&w2+Uylsmyb%0a zQQxvGdN#}pZ(CAVJ4M1tc@`KJR=_gTb&gJK2Ww6dAxU3R7}t(GSTT;f{w48_RM-QI%`2R3jLx;r={C2MY}hen zRs{t7tk@=3r%=}EX-agNq`l$4_hxKaBcP&x%0*;RdH`dWrk-Z?V-RqgNp57cpw72U zhUhoNw;+6?Q8rQ;^ud3>s}1}txKP*;o%&_s3?W4;NEN$HBtsemwpQ?NS5CXc@B@bP z_V_B@qRgEw)-#1u>;GJeQaU8oPC#lTytweg)^EKWW_q!$Y!1su^P&RGHcT_)^}0AI zRfI~0W3Ld*R}`MVuHJQM1wD3Otjcb_owWZJ;aU3r_zPhaxujSCk;FnT1<9(X8F`+> zS^~iRSsOEOk7HfPu947b;nQ7vwm%ZQ5(_W~>{y37Y`{DWH}cLKNq$9V?Y2p*FjJ6m zvVo=yC0$bCZ=qyZQ#dE`w8kP67e34H&ZpDXnNrmrV`4he%CZ-fH*RNsUS9Cag<`4J z#f=2caZ3sunK=CRG#hRuh~J^->h%2si@(Wpn!r)p0G8hX>n@DX%v)>^#(A#`*|4l# z^}vx7N~6sQ#u6x+n-Sjnluae~lZ|5ZO`&1tr@bF`#E!|}k=tUrcNgE*xfEeKlRxp~ zuDhJYHn0gDd~$<9WPbR)ESoAkdSGxSI?vJ4`np#(HcgDn#&2a8=ThXEx;VE_m^^oV zkaqjhGY>xC>2GyqM(JQE^6a18^VKk zJNk}vbr8>2NId7MCu@TPJ&-GcAk6X+1JI>KH;uScql%jZWHCgf)}|<6As7XKkwN9yTYcFTR85R*PVy$tB~lrO zW|1J?h26m0!o4tIy8H3OcEh9O8(QKQ?K;^)a)|wkAjUr~#}$XPYbK;tJk*Jea{>A+ z50e9!z$%Ouw81dSx2YShTftMoRlS>LVH@q8zU`5#G*=L^#noBvtQ1`7j!uFgXtCPQ z+Z8*c@g&0YP0*i+2Nb5@){)NXiwh)km3%shbv(sQr=&wEGA53B3O;#13(QVHIwx)Y zYmNF=8()oDyLsArW{*gnDapot9~e`<6)TpdFi{u!kV87$&B{P8E3uz(6524r9k=C^ zZV;228CH4qS+ExNrVc0I9+{6jv7hC<&`3~}Dt$4bH9>>mS|tD9&FO=1COPN8o`=0` z|5p@srh$^s$DN&>I&U1M&!+|+`DB?f&tes1gP)C3jZm(0{uuraF5^HdZ>7V)5vdpL z#pz*Q@0C~CB;3`wvXg!#LJp&F+A@bI@Ul~IV})sK5iCG|*>rc`iaQ>Ip(o?w;(vm| zboox~R7!{A3|)v$f=yASCZc-<(7zy8av)0-yo)AAjZNP2+uPv7IGpD0sgF5SIkWVR zB9zzr!`p>JlUpH6LOAyV(1$K_O$c*Q0xr^qZqAa7Upc_PwX94GcGe(N%gqMc80-FtJ)@ywXVi>cD$EL~2EQ6k>)@EPXq(3qk+ z-7>>;Ku2xq{Cru20n=2h2{OBE)94*db@9VPW$WoC=7!Lj?dHt>@apH^C&)Txa@6YP z!714WP6XL){Ki_}j1m?1p73OetNLQA(p_Ax_L=zNd`{UoZ3_DtM#X9xD+>j!WK9A>&gVUSaxIsKgKFNAh`o#QUUP z>mTgjUn(-}^mF+&6_##Oh{>dpLD=%{%^77O`CbuKL$*vGvgtr*{b@$jTM3MfCNM8j z9;mP5V#+50ZjTh+6e$(n{?`C9?bgxo?&_Y#-(#zAkX9|iBT*juPcT(OtD-so$6r9r zD@Z&u%o(-Pd485%td!nB<+QebH^SOl&T=~PpBHgWgckws8-HC$^ zw~~7|8#uc7;x#$Ld0e0wmBJ$+W+2j62%WL-=2>pS7+{xie}`|gLdR^F;lqjWbfUCB zI-Gi3q?9JRdx>aQ+(t<+{Kfh1I4D|P1i`ZL!%Z@r$X*E+NF2*e}%VXneEH)%yw;?TqECNFpd%X0DXJ^ zCKf(nD9)hTq?!9p1@SZpLuO!=!vlwGRsFP>&8HG`M7v30tif|VKXP=#i}FjaIa28c z<1i*NFTFfg_5AWJTtLl%q<3Navl`Gj=ZW!5y7w6vdtxAbc61BPX?t)x7uCY6rZ~eU z{!E#e!>|~6Zc{mGMRljt-!XEGt2H!(j3EkU;)>1*XuYOyY=!4I18Lw zo}=nuaZj`MY3w>oWdr;H^uLwL32t4xBU=M#OjV@afa02;G9Ey&I zinOQ2wAIy6MFzOeur_|1;`tF8l!)-I!hl_g2U8fwrUk$9narcSQNFys{=*bnlRbQ} z>cn__X>z}t8~s{RfcDZd6vfBXuK`B{HosV_XjjwRGT z%yxD(-pQ0PZcpL-DWm;v^~u%hoGlfc<3GFEqe3XtHvE!3g@KL#G^=2$^~*CD*^zl9_xXqP9f=yp6>5_5~$EXlH+S@Jpx*}mz5P~{A^fJ$|vnrl<{M!Dg|l;?Z45p z4=@YvTCT>0&bA!4`V*J^o=$+5SpblG0`qf5&p9^9j)KvnN225uq^i1EX6>y&J^^a= zUHs|UWb^Nj;_kRd?o(d~z7HLIhrbo{z2+5w1XTzaY3^D~08r##YiW7(b+_mCbu_Cn zhq8Oty%B)I&CHEGzAVZSr)G8=)Y!L$Ia>)hi^n0~VzOPiE~C`BG+EzbOFWrZtaWa+Ge$xL#5eV%gulufMki zvJHOf9ip1oF4k;!ji)h&upkBl?af;lp@6I|dC%C~!2Y?^!!|8mMb zZklp!Va5Bio#OqMHov!*yVg}>QCaxbTY+1w%C^g4z&`w1#F=JOL28Pbp3<9h8YItD z>#S)D`&DM?O2s|hXC5tK-Oj0+_ZZx%VODLb0y!~xW0Om-T`a@dAo(pqPlUNb5**=n z&xtjkGHr6H=b?B9KVzh_SQ(^`^?iIQ zKkU1-8Yw)t;}r6&gCS&LFRXr}ShU_O_K8j7QW=u)sam!}&H(4@DOvAHeu12qEq-w; zrVm^;0>i*ijtlh+0U^}z&Vbmma-(sf;~s;oEg@NPuEQ>QGTh2lJXx~t)jnbci98+xQg4(>i||s!6>T=sRu<+j`p`3`n#ymR)k%&M{LKMF6a9*La6w zR@Ysr?J|6KWWDkX<}39VsHTmHNcDQ=Q2l@bGT{95(xfECqtf7wqKMNU?;(Mv!*z;H zSZ6jM{J4h%|NCA_wO$)^gruUl({dw&VaF!bu^A>f{3O8cfL;?e^3lV7)*CwSOyQIf zlni3;&D%ax02lzv&6eT*CRKv>94W}naXz#)5BlY3?)D0gO7@EYnS~v{YpDOGWHmB_ zGM#|T8y88LFb(||&gyU?(}MIXd;^92=ORMZWaXK@Y?X`72%!(U6vcyX5IUkpMRk(A zd?hsp-V^(Y4Lkpy0cYdsR^p-n_lwtsb(EmJ!?LO($e}4=n_`;5m~+$Y@Cd|Piy*HJ z2m?$o`T45s*IEXD4acbgosaB44_WcoOF!kvEOuiAN{5uO32uq3Evx2_QgjQ6UIg&R zMP#fTqjc=0&)cDkOiYFImvvg$B*n#d{u^!0JIl@!`Y~g5xJ?YC!9~~ODR-yPzd@k% zgJ@0gSl=EWc*63TeRN69Wv?;CuKtTZjFWY|hv>(j8Y7>FGss3prUJsR^ta+%*hHVn z%4n-M_e8?CK2CXnC}37w>*3qL-z|f#*TMW;(}5Uss+`>WH?e2%M~Pz(wX%b%g55{h zY8MrKG#%+c#46<}hvq-X=sO5SvCpd70vz5vF`h0=x}k5+7T~&F3_TkmAx(q=#ej zI&y)oL-CJwpj%a5UJ!DVu+sdb<~jiSt^d^>T2+wAfsMhUkpf`wP{{T=WNbXH!B@Mf zROzy>M$fcoE{Tf8 zQ$4;EG=Po4Wmtg5$XE*_cRY^FoI_$opjhXc6fuz0bddSKtgs9S*ED_I4pH39m+OK2eA!2jx-_-RWK`0DsfeeIE?9xl%Wiyod zUsD?z;H|zp>0**Pi^Z`HW0N{Ay#?zkSMja{GA)-p5A`BNol#ci4#CAwOZS6;`-0BM zt0T`L{D_)HsJg;PmlYGjSBfTZ?gnRkoeuyr@k)HrVtw-=)d$IuYU? zICkC=yjpNNNApyj|7R&_pANj=XGfsxO#9q8Sy|pq@ zfV5X3vvoOMJwTy{Kw4e4SGwA%@57nA9bodv+PwX)zc#0r&Cw;2CHaIJRq)==*d#=m zmDMujhzw+tZB1GpmHY6Mt}+b|iCHp3Vj`4AiT{`b0CWnESsDE?zPX(jOskDLOb|ec zJvF?bqa}^WQRUWRl|j#K+3GaDVCpL-8oiYf>Wtdaiml^Pyv{{a8*UlLZ^}`924R^v47?hO)@JDAW{1nC!XLzU z?=vI~7bbp8^y7xHb=-fvI(6-*LsR~mgaouPGm8gDqrLpVO-eY~`^#Oe@H|AH4Yp&B z;d1S^iVxvWOmTHV61kp~1h$rQtcul3$ZTy|p*P-J#LH$EH5m-_OL5^l1Ljx#hec#@&q-!?jqi$T2%`?ZSzG90agBI!YovOgS|gP!zyXd=d^^vH zsp}RTnd)vZ-fhiWJ;PT+qb#iE?8PpuulJ`n%G|_ zO4`-dGur1)bs%Z1Mymwal?x7S5CjI9_#cNa-Pavp{q&eWhS7ul+*t#^u(#&puNnQv zSx=bUoOO;x*vTHF90L%a@or}&F2>7=+E2iF^r}6_mGs=WBDW;)SIbVnm&Nrq>IT$= z-At5K|D)<}q9v?PMUDlW`n^=kx4C?x6=dL^o6ShJqRe}vXaCS2jjlq} z;$nZv{M=|hFhI4+wYw=EVs4-R)+nTN92$ui^=c?rs^-f~b#U^V6V|^*LR}}C;S4x- z1Lcg3a{!GYBbs2VF!XRRB!ER`;~8nm+bUEvG{J5MMrRsZ{NG*!^{Z(yHP;<~vbN1; z{(AKzCgH2}k)N>s%u=etpC5BIU>v@(u89!c{UZAA{jmx(QLM+=P8s9Da^CgwwCzfn z@>Uc@TNM8RSE!#mzT?~BvXE$?0fg>5_D*IDQKbu*Q;Eu2dF(D2zp`mg7(+`)oaV;c zIPIKl7B?JvM4LLF;b6?#a+VI;sitLE5FOfYG~ZuFy15DtE)I~}W(K#%;k=>7aJM~{ z8s2aG?5wtUhcO??+A`=5^9cmzZdvhVI=|9PHUl3t)y*D6cSq69PTBpEKZ&}yZPEX=xuxMcJa%Ny07bQDWqpT_1V-0j0xc>W2F}w#{c8+sEY@nxQyWjHM zb7~?V73~a%Gp}p;cZ}RDr;A(OeA2kqQLa^GAnADTNXLx0R0#egqXUVnhzf0|KD8Xr zke6LrCs<@!Tl`~s=Xb%HZMtWu62xSD1x^#z(t&oL@3s$ZOMHS z%|F1Mo7wzstT6Iu(I0gW^6vYlzzPzQmph||kTjBo=8Oh%$mnPFFBFrC=i3S29w&3w zWYbcTMU>o9-s+Y(PJ0Lv@H*X6iEtWbPn3wp#HP0FBvPv}qW>l<_P^#|-9LT!6-V@2cnc;o~a* z7u){8ojROgY} z9(~iF5H$LgH%L;Ems7y~C)K2p(TvHbKj9#Axk45%YTAbG!Ut1^1`&v=?M6>f_dBLj zexWwj3c6v_PF+~PNJ{zVf?ov>FM@ezLDc&P>&~sVK80{1Z~3hc#c2p=P%1U)5U6mn z{x4=Frrk9q^L2A@nl4)N+ws#y?)z!gn{xCK{Ur0mWjV?$NlL_UgA?m;qZ>}WDyj6Z zO%L_zhVyxjKq>Rl!he-6J*Fr!q-VnOj@D%XMh2=@gVR!$@O9(kyyB&E+JEjodZ1sA zJJAM7)M=?N@$4Q}UbS1Kl}utmuqL^;jveHcE7#p)(!hqP{R5UjLl;8I?!Dv}ux}N! z>Z~nN1^S5TkN1Cg-ZSO(75N?T|BfWK=SA)b+q#)Iv^1+AF8R$gy|`Asl4zZUgWvbf zSd#TGH|`hOyz=s>d6KM4g5>Ynt%XX8G`&yJq8K>j?-9Q3T8pZ0uKoPoeve#U>4q1`2z%=^+PBQh- zq2l$SJzhm3$#nSrD6XqD%G-=;JzT>hiz|YOCBl48*ek|o;`S+8zemUP>k`9T7^fWP zS|HrVTrVq<2k7e>IPCGqy6c0^&lMHY34SUS`;YO=gQE_m3a;30Uo^~Bi#|^5t@^rL z8wTitB$CY6>9$|+eH`da+tw;r zHHC5`@O3)=@>KQnRCVno((USNFLe1-<(zPPiO49Hm6soz1Z&c=_)?;2XJ{RaVuk*) z3N}v>K7LxPG!zetcEbr{6a|rAWW;%Z*TUv7Yng(uhI!rd$^Mp$KhbTmT5%MB@Zz1cg-R@}&+V%xy z7}cAS+TqGrrv#R^5mDlsGstTt4HK|H;jeUl{4`^WR8a zL~T zLA84cs#IyUG-f|s+luGAhUM4+DO6T;`y|c!zKKlG{87KKucDnQsDuLwr}gs&h3vM` z((xU!(UD4YJM>Jvc5D7YriacmaO`OOpZR(2>sh^@g`rQiF>P4h_j{TNwrX4?qzkYB z9f?b+=Nsw8zpCou=0Be=?T>ul=-#pYr5NDojg^2H05bob5!$59oCiIm`ud&yH*;8 zg6Ca`dwBt(sMMa!({2p$(2+<7bK-IEhX!l@H%H(RIgknMV7}J!o9G0MeNSZen&RKn zXM9MepW6P{E?^U`gcITN5$L%!IGJoP^eoiisgwH6uK6xNugQFj0|o7*Os zJYopJ%Ut`@sN(LoP*a>ri>~ULg!30FwAIH|mXW>>m9Ib7vRh$`F`t9-8RjT|GDA;Qp*uN{{ajjOs2HFP99SO59>`VAl*_kuTW{L1)6x(zav9An8l`049)+p6~6 z<9a|3URXUZeg$R|mrX=VD}E~w)OY~u>jQ+By~ zZ#3dTOW47oz!;*E23~vCAbxs$XeYS_M4PG}kl;%zsgK;$esGhTN8|qggh`~g;aTWXz+X;~v9oY;g*rNAF ztOL)aJ_Ri7>dBGB(Ze<7Fh?i75tidS3sST;j0mXT^WiZ08=;R4%S~CZ2#-nWKDjGt zRU>a?U{84<_RnJE?mIWHT}2nW&VS9L+QU#nWGJ(Ph@qOFV*4c;xx4BQ3E&Fvpw?^CDYbQyQxDTx7}3B)3DKpALv}nW%|q>j7E#n zg!>2kZY4F;RBMEQB7-4SEB9wzdK$_4elgkcfa;1B=P zp##2a-z|QOY0SsqfBztgwtFBGLBO4ca)e-#Pc?(yRjI*l!M@e+{NYP)N{!FMWi%@| z7`M*}rfzIg0R<9D+M}V;8_lc}{V;(=C5gT_h;)Ctc+WN8~6D8e^MtoufwWOjy@cl!$O_gyet16AO(e=&j8Ra zC!LR}?bb(h?bi46J+k!(jExRDHUr02-$lbGqlm#`q3@3V*J-a+=YOlPAoD35Yt yiH@sIh^{$>WX72NIrjf2g#XV>Nl&3Y`}<&}R#gQ@|Ax(%!^ul4OI1o31^o}L|JGyx literal 0 HcmV?d00001 diff --git a/website/www/site/static/js/hero/hero-desktop.js b/website/www/site/static/js/hero/hero-desktop.js new file mode 100644 index 000000000000..4833284937ec --- /dev/null +++ b/website/www/site/static/js/hero/hero-desktop.js @@ -0,0 +1,21 @@ +// Licensed under the Apache License, Version 2.0 (the 'License'); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +var animationData = {"v":"5.7.1","fr":30,"ip":0,"op":120,"w":1440,"h":536,"nm":"opt2","ddd":0,"assets":[],"layers":[{"ddd":0,"ind":2,"ty":4,"nm":"Shape Layer 1","sr":1,"ks":{"o":{"a":0,"k":24,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[62.997],"e":[20]},{"t":77}],"ix":10},"p":{"a":0,"k":[2112,250,0],"ix":2},"a":{"a":0,"k":[1228.201,-1.163,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[72.266,-33.656],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":3,"ty":4,"nm":"Shape Layer 6","sr":1,"ks":{"o":{"a":0,"k":82,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[-35],"e":[-20]},{"t":56}],"ix":10},"p":{"a":0,"k":[-1802,346,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.341176470588,0.043137254902,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":4,"ty":4,"nm":"Shape Layer 7","sr":1,"ks":{"o":{"a":0,"k":66,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[-53],"e":[-34]},{"t":76}],"ix":10},"p":{"a":0,"k":[-1690,996,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":5,"ty":4,"nm":"Shape Layer 2","sr":1,"ks":{"o":{"a":0,"k":24,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-6,"s":[-46],"e":[20]},{"t":67}],"ix":10},"p":{"a":0,"k":[1436,1008,0],"ix":2},"a":{"a":0,"k":[1171.735,-35.884,0],"ix":1},"s":{"a":0,"k":[100,78.683,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[72.266,-33.656],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":6,"ty":4,"nm":"Shape Layer 4","sr":1,"ks":{"o":{"a":0,"k":20,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[0.759]},"t":-5,"s":[2],"e":[-27]},{"t":56}],"ix":10},"p":{"a":0,"k":[-188,1280,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":8,"ty":4,"nm":"Shape Layer 8","sr":1,"ks":{"o":{"a":0,"k":57,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-7,"s":[2],"e":[-43]},{"t":56}],"ix":10},"p":{"a":0,"k":[-66,1694,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,81.692,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-152.112,163.925],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":9,"ty":1,"nm":"Orange Solid 3","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[720,268,0],"ix":2},"a":{"a":0,"k":[741.5,447,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"sw":1483,"sh":894,"sc":"#ff570b","ip":0,"op":720,"st":0,"bm":0},{"ddd":0,"ind":11,"ty":1,"nm":"Medium Orange Solid 1","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[720,268,0],"ix":2},"a":{"a":0,"k":[741.5,447,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"sw":1483,"sh":894,"sc":"#f87924","ip":0,"op":720,"st":0,"bm":0}],"markers":[]}; + +var animation = bodymovin.loadAnimation({ + container: document.getElementById("hero-desktop"), + renderer: "svg", + loop: false, + autoplay: true, + animationData: animationData, +}); diff --git a/website/www/site/static/js/hero/hero-mobile.js b/website/www/site/static/js/hero/hero-mobile.js new file mode 100644 index 000000000000..979a5ec9dccc --- /dev/null +++ b/website/www/site/static/js/hero/hero-mobile.js @@ -0,0 +1,21 @@ +// Licensed under the Apache License, Version 2.0 (the 'License'); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +var animationData = {"v":"5.7.1","fr":30,"ip":0,"op":120,"w":375,"h":462,"nm":"opt2","ddd":0,"assets":[],"layers":[{"ddd":0,"ind":2,"ty":4,"nm":"Shape Layer 1","sr":1,"ks":{"o":{"a":0,"k":24,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[62.997],"e":[20]},{"t":77}],"ix":10},"p":{"a":0,"k":[1635.5,319,0],"ix":2},"a":{"a":0,"k":[1228.201,-1.163,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[72.266,-33.656],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":3,"ty":4,"nm":"Shape Layer 6","sr":1,"ks":{"o":{"a":0,"k":82,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[-35],"e":[-20]},{"t":56}],"ix":10},"p":{"a":0,"k":[-2082.5,375,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.341176470588,0.043137254902,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":4,"ty":4,"nm":"Shape Layer 7","sr":1,"ks":{"o":{"a":0,"k":66,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-5,"s":[-53],"e":[-34]},{"t":76}],"ix":10},"p":{"a":0,"k":[-1990.5,983,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":5,"ty":4,"nm":"Shape Layer 2","sr":1,"ks":{"o":{"a":0,"k":24,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-6,"s":[-46],"e":[20]},{"t":67}],"ix":10},"p":{"a":0,"k":[1117.5,959,0],"ix":2},"a":{"a":0,"k":[1171.735,-35.884,0],"ix":1},"s":{"a":0,"k":[100,78.683,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[72.266,-33.656],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":6,"ty":4,"nm":"Shape Layer 4","sr":1,"ks":{"o":{"a":0,"k":20,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[0.759]},"t":-5,"s":[2],"e":[-27]},{"t":56}],"ix":10},"p":{"a":0,"k":[-484.5,1051,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,135.77,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-5.792,59.506],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":8,"ty":4,"nm":"Shape Layer 8","sr":1,"ks":{"o":{"a":0,"k":66,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.34],"y":[1]},"o":{"x":[0.22],"y":[1]},"t":-7,"s":[2],"e":[-43]},{"t":56}],"ix":10},"p":{"a":0,"k":[-542.5,1281,0],"ix":2},"a":{"a":0,"k":[-1078.126,-3.829,0],"ix":1},"s":{"a":0,"k":[154.688,81.692,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ty":"rc","d":1,"s":{"a":0,"k":[2315.531,506.688],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"r":{"a":0,"k":0,"ix":4},"nm":"Rectangle Path 1","mn":"ADBE Vector Shape - Rect","hd":false},{"ty":"fl","c":{"a":0,"k":[1,0.76862745098,0,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[-152.112,163.925],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Rectangle 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":-6,"op":714,"st":-6,"bm":0},{"ddd":0,"ind":9,"ty":1,"nm":"Orange Solid 3","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,231,0],"ix":2},"a":{"a":0,"k":[741.5,447,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"sw":1483,"sh":894,"sc":"#ff570b","ip":0,"op":720,"st":0,"bm":0},{"ddd":0,"ind":11,"ty":1,"nm":"Medium Orange Solid 1","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,231,0],"ix":2},"a":{"a":0,"k":[741.5,447,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"sw":1483,"sh":894,"sc":"#f87924","ip":0,"op":720,"st":0,"bm":0}],"markers":[]}; + +var animation = bodymovin.loadAnimation({ + container: document.getElementById("hero-mobile"), + renderer: "svg", + loop: false, + autoplay: true, + animationData: animationData, +}); diff --git a/website/www/site/static/js/hero/lottie-light.min.js b/website/www/site/static/js/hero/lottie-light.min.js new file mode 100644 index 000000000000..feb17961b669 --- /dev/null +++ b/website/www/site/static/js/hero/lottie-light.min.js @@ -0,0 +1,18 @@ +// TODO [website-revamp]: +// License header should be added + +(typeof navigator !== "undefined") && (function(root, factory) { + if (typeof define === "function" && define.amd) { + define(function() { + return factory(root); + }); + } else if (typeof module === "object" && module.exports) { + module.exports = factory(root); + } else { + root.lottie = factory(root); + root.bodymovin = root.lottie; + } +}((window || {}), function(window) { + "use strict";var h,e="http://www.w3.org/2000/svg",A="",s=-999999,i=!0,b=(/^((?!chrome|android).)*safari/i.test(navigator.userAgent),Math.round,Math.pow),k=Math.sqrt,f=(Math.abs,Math.floor),m=(Math.max,Math.min),a={};!function(){var t,e=["abs","acos","acosh","asin","asinh","atan","atanh","atan2","ceil","cbrt","expm1","clz32","cos","cosh","exp","floor","fround","hypot","imul","log","log1p","log2","log10","max","min","pow","random","round","sign","sin","sinh","sqrt","tan","tanh","trunc","E","LN10","LN2","LOG10E","LOG2E","PI","SQRT1_2","SQRT2"],s=e.length;for(t=0;t>>=1;return(t+s)/e};return n.int32=function(){return 0|r.g(4)},n.quick=function(){return r.g(4)/4294967296},n.double=n,b(k(r.S),h),(e.pass||s||function(t,e,s,i){return i&&(i.S&&_(i,r),t.state=function(){return _(r,{})}),s?(o[d]=t,e):t})(n,a,"global"in e?e.global:this==o,e.state)},b(o.random(),h)}([],a);var W=function(){var t={getBezierEasing:function(t,e,s,i,a){var r=a||("bez_"+t+"_"+e+"_"+s+"_"+i).replace(/\./g,"p");if(h[r])return h[r];var n=new o([t,e,s,i]);return h[r]=n}},h={};var l=11,p=1/(l-1),e="function"==typeof Float32Array;function i(t,e){return 1-3*e+3*t}function a(t,e){return 3*e-6*t}function r(t){return 3*t}function f(t,e,s){return((i(e,s)*t+a(e,s))*t+r(e))*t}function m(t,e,s){return 3*i(e,s)*t*t+2*a(e,s)*t+r(e)}function o(t){this._p=t,this._mSampleValues=e?new Float32Array(l):new Array(l),this._precomputed=!1,this.get=this.get.bind(this)}return o.prototype={get:function(t){var e=this._p[0],s=this._p[1],i=this._p[2],a=this._p[3];return this._precomputed||this._precompute(),e===s&&i===a?t:0===t?0:1===t?1:f(this._getTForX(t),s,a)},_precompute:function(){var t=this._p[0],e=this._p[1],s=this._p[2],i=this._p[3];this._precomputed=!0,t===e&&s===i||this._calcSampleValues()},_calcSampleValues:function(){for(var t=this._p[0],e=this._p[2],s=0;sn?-1:1,l=!0;l;)if(i[r]<=n&&i[r+1]>n?(h=(n-i[r])/(i[r+1]-i[r]),l=!1):r+=o,r<0||a-1<=r){if(r===a-1)return s[r];l=!1}return s[r]+(s[r+1]-s[r])*h}var M=j("float32",8);return{getSegmentsLength:function(t){var e,s=Ct.newElement(),i=t.c,a=t.v,r=t.o,n=t.i,h=t._length,o=s.lengths,l=0;for(e=0;es[0]||!(s[0]>t[0])&&(t[1]>s[1]||!(s[1]>t[1])&&(t[2]>s[2]||!(s[2]>t[2])&&void 0))}var o,s=function(){var i=[4,4,14];function a(t){var e,s,i,a=t.length;for(e=0;e=r.t-i){a.h&&(a=r),m=0;break}if(r.t-i>t){m=d;break}d=s&&s<=t||this._caching.lastFrame=t&&(this._caching._lastKeyframeIndex=-1,this._caching.lastIndex=0);var i=this.interpolateValue(t,this._caching);this.pv=i}return this._caching.lastFrame=t,this.pv}function c(t){var e;if("unidimensional"===this.propType)e=t*this.mult,1e-5=this.p.keyframes[this.p.keyframes.length-1].t?(e=this.p.getValueAtTime(this.p.keyframes[this.p.keyframes.length-1].t/i,0),this.p.getValueAtTime((this.p.keyframes[this.p.keyframes.length-1].t-.05)/i,0)):(e=this.p.pv,this.p.getValueAtTime((this.p._caching.lastFrame+this.p.offsetTime-.01)/i,this.p.offsetTime));else if(this.px&&this.px.keyframes&&this.py.keyframes&&this.px.getValueAtTime&&this.py.getValueAtTime){e=[],s=[];var a=this.px,r=this.py;a._caching.lastFrame+a.offsetTime<=a.keyframes[0].t?(e[0]=a.getValueAtTime((a.keyframes[0].t+.01)/i,0),e[1]=r.getValueAtTime((r.keyframes[0].t+.01)/i,0),s[0]=a.getValueAtTime(a.keyframes[0].t/i,0),s[1]=r.getValueAtTime(r.keyframes[0].t/i,0)):a._caching.lastFrame+a.offsetTime>=a.keyframes[a.keyframes.length-1].t?(e[0]=a.getValueAtTime(a.keyframes[a.keyframes.length-1].t/i,0),e[1]=r.getValueAtTime(r.keyframes[r.keyframes.length-1].t/i,0),s[0]=a.getValueAtTime((a.keyframes[a.keyframes.length-1].t-.01)/i,0),s[1]=r.getValueAtTime((r.keyframes[r.keyframes.length-1].t-.01)/i,0)):(e=[a.pv,r.pv],s[0]=a.getValueAtTime((a._caching.lastFrame+a.offsetTime-.01)/i,a.offsetTime),s[1]=r.getValueAtTime((r._caching.lastFrame+r.offsetTime-.01)/i,r.offsetTime))}else e=s=n;this.v.rotate(-Math.atan2(e[1]-s[1],e[0]-s[0]))}this.data.p&&this.data.p.s?this.data.p.z?this.v.translate(this.px.v,this.py.v,-this.pz.v):this.v.translate(this.px.v,this.py.v,0):this.v.translate(this.p.v[0],this.p.v[1],-this.p.v[2])}this.frameId=this.elem.globalData.frameId}},precalculateMatrix:function(){if(!this.a.k&&(this.pre.translate(-this.a.v[0],-this.a.v[1],this.a.v[2]),this.appliedTransformations=1,!this.s.effectsSequence.length)){if(this.pre.scale(this.s.v[0],this.s.v[1],this.s.v[2]),this.appliedTransformations=2,this.sk){if(this.sk.effectsSequence.length||this.sa.effectsSequence.length)return;this.pre.skewFromAxis(-this.sk.v,this.sa.v),this.appliedTransformations=3}if(this.r){if(this.r.effectsSequence.length)return;this.pre.rotate(-this.r.v),this.appliedTransformations=4}else this.rz.effectsSequence.length||this.ry.effectsSequence.length||this.rx.effectsSequence.length||this.or.effectsSequence.length||(this.pre.rotateZ(-this.rz.v).rotateY(this.ry.v).rotateX(this.rx.v).rotateZ(-this.or.v[2]).rotateY(this.or.v[1]).rotateX(this.or.v[0]),this.appliedTransformations=4)}},autoOrient:function(){}},F([w],i),i.prototype.addDynamicProperty=function(t){this._addDynamicProperty(t),this.elem.addDynamicProperty(t),this._isDirty=!0},i.prototype._addDynamicProperty=w.prototype.addDynamicProperty,{getTransformProperty:function(t,e,s){return new i(t,e,s)}}}();function z(){this.c=!1,this._length=0,this._maxLength=8,this.v=x(this._maxLength),this.o=x(this._maxLength),this.i=x(this._maxLength)}z.prototype.setPathData=function(t,e){this.c=t,this.setLength(e);for(var s=0;s=this._maxLength&&this.doubleArrayLength(),s){case"v":r=this.v;break;case"i":r=this.i;break;case"o":r=this.o}(!r[i]||r[i]&&!a)&&(r[i]=Pt.newElement()),r[i][0]=t,r[i][1]=e},z.prototype.setTripleAt=function(t,e,s,i,a,r,n,h){this.setXYAt(t,e,"v",n,h),this.setXYAt(s,i,"o",n,h),this.setXYAt(a,r,"i",n,h)},z.prototype.reverse=function(){var t=new z;t.setPathData(this.c,this._length);var e=this.v,s=this.o,i=this.i,a=0;this.c&&(t.setTripleAt(e[0][0],e[0][1],i[0][0],i[0][1],s[0][0],s[0][1],0,!1),a=1);var r,n=this._length-1,h=this._length;for(r=a;r=d[d.length-1].t-this.offsetTime)i=d[d.length-1].s?d[d.length-1].s[0]:d[d.length-2].e[0],r=!0;else{for(var c,u,g=m,y=d.length-1,v=!0;v&&(c=d[g],!((u=d[g+1]).t-this.offsetTime>t));)g=u.t-this.offsetTime)p=1;else if(ti+s);else p=h.s*a<=i?0:(h.s*a-i)/s,f=h.e*a>=i+s?1:(h.e*a-i)/s,o.push([p,f])}return o.length||o.push([0,0]),o},G.prototype.releasePathsData=function(t){var e,s=t.length;for(e=0;ee.e){s.c=!1;break}e.s<=c&&e.e>=c+n.addedLength?(this.addSegment(m[i].v[a-1],m[i].o[a-1],m[i].i[a],m[i].v[a],s,h,g),g=!1):(l=mt.getNewSegment(m[i].v[a-1],m[i].v[a],m[i].o[a-1],m[i].i[a],(e.s-c)/n.addedLength,(e.e-c)/n.addedLength,o[a-1]),this.addSegmentFromArray(l,s,h,g),g=!1,s.c=!1),c+=n.addedLength,h+=1}if(m[i].c&&o.length){if(n=o[a-1],c<=e.e){var y=o[a-1].addedLength;e.s<=c&&e.e>=c+y?(this.addSegment(m[i].v[a-1],m[i].o[a-1],m[i].i[0],m[i].v[0],s,h,g),g=!1):(l=mt.getNewSegment(m[i].v[a-1],m[i].v[0],m[i].o[a-1],m[i].i[0],(e.s-c)/y,(e.e-c)/y,o[a-1]),this.addSegmentFromArray(l,s,h,g),g=!1,s.c=!1)}else s.c=!1;c+=n.addedLength,h+=1}if(s._length&&(s.setXYAt(s.v[p][0],s.v[p][1],"i",p),s.setXYAt(s.v[s._length-1][0],s.v[s._length-1][1],"o",s._length-1)),c>e.e)break;i=c.length&&(f=0,c=u[m+=1]?u[m].points:b.v.c?u[m=f=0].points:(l-=o.partialLength,null)),c&&(d=o,g=(o=c[f]).partialLength));z=D[a].an/2-D[a].add,S.translate(-z,0,0)}else z=D[a].an/2-D[a].add,S.translate(-z,0,0),S.translate(-k[0]*D[a].an/200,-k[1]*L/100,0);for(D[a].l/2,F=0;Fe));)s+=1;return this.keysIndex!==s&&(this.keysIndex=s),this.data.d.k[this.keysIndex].s},ct.prototype.buildFinalText=function(t){for(var e,s=L.getCombinedCharacterCodes(),i=[],a=0,r=t.length;athis.minimumFontSize&&M=u(h)&&(n=d(0,c(t-h<0?c(o,1)-(h-t):o-t,1))),r(n));return n*this.a.v},getValue:function(t){this.iterateDynamicProperties(),this._mdf=t||this._mdf,this._currentTextLength=this.elem.textProperty.currentData.l.length||0,t&&2===this.data.r&&(this.e.v=this._currentTextLength);var e=2===this.data.r?1:100/this.data.totalChars,s=this.o.v/e,i=this.s.v/e+s,a=this.e.v/e+s;if(at-this.layers[e].st&&this.buildItem(e),this.completeLayers=!!this.elements[e]&&this.completeLayers;this.checkPendingElements()},wt.prototype.createItem=function(t){switch(t.ty){case 2:return this.createImage(t);case 0:return this.createComp(t);case 1:return this.createSolid(t);case 3:return this.createNull(t);case 4:return this.createShape(t);case 5:return this.createText(t);case 6:return this.createAudio(t);case 13:return this.createCamera(t)}return this.createNull(t)},wt.prototype.createCamera=function(){throw new Error("You're using a 3d camera. Try the html renderer.")},wt.prototype.createAudio=function(t){return new te(t,this.globalData,this)},wt.prototype.buildAllItems=function(){var t,e=this.layers.length;for(t=0;tt?!0!==this.isInRange&&(this.globalData._mdf=!0,this._mdf=!0,this.isInRange=!0,this.show()):!1!==this.isInRange&&(this.globalData._mdf=!0,this.isInRange=!1,this.hide())},renderRenderable:function(){var t,e=this.renderableComponents.length;for(t=0;tthis.animationData.op&&(this.animationData.op=t.op,this.totalFrames=Math.floor(t.op-this.animationData.ip));var e,s,i=this.animationData.layers,a=i.length,r=t.layers,n=r.length;for(s=0;sthis.timeCompleted&&(this.currentFrame=this.timeCompleted),this.trigger("enterFrame"),this.renderFrame()},he.prototype.renderFrame=function(){if(!1!==this.isLoaded)try{this.renderer.renderFrame(this.currentFrame+this.firstFrame)}catch(t){this.triggerRenderFrameError(t)}},he.prototype.play=function(t){t&&this.name!=t||!0===this.isPaused&&(this.isPaused=!1,this.audioController.resume(),this._idle&&(this._idle=!1,this.trigger("_active")))},he.prototype.pause=function(t){t&&this.name!=t||!1===this.isPaused&&(this.isPaused=!0,this._idle=!0,this.trigger("_idle"),this.audioController.pause())},he.prototype.togglePause=function(t){t&&this.name!=t||(!0===this.isPaused?this.play():this.pause())},he.prototype.stop=function(t){t&&this.name!=t||(this.pause(),this.playCount=0,this._completedLoop=!1,this.setCurrentRawFrameValue(0))},he.prototype.goToAndStop=function(t,e,s){s&&this.name!=s||(e?this.setCurrentRawFrameValue(t):this.setCurrentRawFrameValue(t*this.frameModifier),this.pause())},he.prototype.goToAndPlay=function(t,e,s){this.goToAndStop(t,e,s),this.play()},he.prototype.advanceTime=function(t){if(!0!==this.isPaused&&!1!==this.isLoaded){var e=this.currentRawFrame+t*this.frameModifier,s=!1;e>=this.totalFrames-1&&0=this.totalFrames?(this.playCount+=1,this.checkSegments(e%this.totalFrames)||(this.setCurrentRawFrameValue(e%this.totalFrames),this._completedLoop=!0,this.trigger("loopComplete"))):this.setCurrentRawFrameValue(e):this.checkSegments(e>this.totalFrames?e%this.totalFrames:0)||(s=!0,e=this.totalFrames-1):e<0?this.checkSegments(e%this.totalFrames)||(!this.loop||this.playCount--<=0&&!0!==this.loop?(s=!0,e=0):(this.setCurrentRawFrameValue(this.totalFrames+e%this.totalFrames),this._completedLoop?this.trigger("loopComplete"):this._completedLoop=!0)):this.setCurrentRawFrameValue(e),s&&(this.setCurrentRawFrameValue(e),this.pause(),this.trigger("complete"))}},he.prototype.adjustSegment=function(t,e){this.playCount=0,t[1]t[0]&&(this.frameModifier<0&&(this.playSpeed<0?this.setSpeed(-this.playSpeed):this.setDirection(1)),this.timeCompleted=this.totalFrames=t[1]-t[0],this.firstFrame=t[0],this.setCurrentRawFrameValue(.001+e)),this.trigger("segmentStart")},he.prototype.setSegment=function(t,e){var s=-1;this.isPaused&&(this.currentRawFrame+this.firstFramee&&(s=e-t)),this.firstFrame=t,this.timeCompleted=this.totalFrames=e-t,-1!==s&&this.goToAndStop(s,!0)},he.prototype.playSegments=function(t,e){if(e&&(this.segments.length=0),"object"==typeof t[0]){var s,i=t.length;for(s=0;s + * License: MIT + * Released on: 2020-11-10 + */ + +!function(t,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(t=t||self).KeenSlider=n()}(this,(function(){"use strict";function t(t,n,e){return n in t?Object.defineProperty(t,n,{value:e,enumerable:!0,configurable:!0,writable:!0}):t[n]=e,t}function n(t,n){var e=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(t,n).enumerable}))),e.push.apply(e,r)}return e}function e(e){for(var r=1;rt.length)&&(n=t.length);for(var e=0,r=new Array(n);e1&&void 0!==arguments[1]?arguments[1]:document;return"function"==typeof t?o(t()):"string"==typeof t?o(n.querySelectorAll(t)):t instanceof HTMLElement!=!1?[t]:t instanceof NodeList!=!1?t:[]}function u(t,n,e){return Math.min(Math.max(t,n),e)}return Math.sign||(Math.sign=function(t){return(t>0)-(t<0)||+t}),function(t){var n,i,o,c,f,s,l,d,h,v,p,m,b,g,y,w,M,O,S,j,A,x,k,E,P,T,D,C,L,X,Y,z,H=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},I="data-keen-slider-moves",V="data-keen-slider-v",q=[],F=null,W=!1,_=!1,K=0,N=[];function R(t,n,e){var r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};t.addEventListener(n,e,r),q.push([t,n,e,r])}function U(t){if(O&&S===J(t)&&ut()){var e=Z(t).x;if(!nt(t)&&E)return B(t);E&&(Wt(),j=e,n.setAttribute(I,!0),E=!1),t.cancelable&&t.preventDefault(),Xt(k(j-e,$t),t.timeStamp),j=e}}function $(t){O||!ut()||tt(t.target)||(O=!0,E=!0,S=J(t),nt(t),ht(),M=v,j=Z(t).x,Xt(0,t.timeStamp),ot("dragStart"))}function B(t){O&&S===J(t,!0)&&ut()&&(n.removeAttribute(I),O=!1,mt(),ot("dragEnd"))}function G(t){return t.changedTouches}function J(t){var n=arguments.length>1&&void 0!==arguments[1]&&arguments[1],e=n?G(t):Q(t);return e?e[0]?e[0].identifier:"error":"default"}function Q(t){return t.targetTouches}function Z(t){var n=Q(t);return{x:lt()?n?n[0].screenY:t.pageY:n?n[0].screenX:t.pageX,timestamp:t.timeStamp}}function tt(t){return t.hasAttribute(w.preventEvent)}function nt(t){var n=Q(t);if(!n)return!0;var e=n[0],r=lt()?e.clientY:e.clientX,i=lt()?e.clientX:e.clientY,o=void 0!==A&&void 0!==x&&Math.abs(x-i)<=Math.abs(A-r);return A=r,x=i,o}function et(t){ut()&&O&&t.preventDefault()}function rt(){R(window,"orientationchange",Pt),R(window,"resize",(function(){return Et()})),R(n,"dragstart",(function(t){ut()&&t.preventDefault()})),R(n,"mousedown",$),R(n,"mousemove",U),R(n,"mouseleave",B),R(n,"mouseup",B),R(n,"touchstart",$,{passive:!0}),R(n,"touchmove",U,{passive:!1}),R(n,"touchend",B,{passive:!0}),R(n,"touchcancel",B,{passive:!0}),R(window,"wheel",et,{passive:!1})}function it(){q.forEach((function(t){t[0].removeEventListener(t[1],t[2],t[3])})),q=[]}function ot(t){w[t]&&w[t]($t)}function at(){return w.centered}function ut(){return void 0!==i?i:w.controls}function ct(){return w.loop}function ft(){return w.rtl}function st(){return!w.loop&&w.rubberband}function lt(){return!!w.vertical}function dt(){P=window.requestAnimationFrame(vt)}function ht(){P&&(window.cancelAnimationFrame(P),P=null),T=null}function vt(t){T||(T=t);var n=t-T,e=pt(n);if(n>=C)return Xt(D-X,!1),z?z():void ot("afterChange");var r=Yt(e);if(0===r||ct()||st()||Y){if(0!==r&&st()&&!Y)return Mt();X+=e,Xt(e,!1),dt()}else Xt(e-r,!1)}function pt(t){return D*L(t/C)-X}function mt(){switch(ot("beforeChange"),w.mode){case"free":yt();break;case"free-snap":wt();break;case"snap":default:bt()}}function bt(){gt((1===l&&0!==p?M:v)+Math.sign(p))}function gt(t,n){var e=arguments.length>2&&void 0!==arguments[2]?arguments[2]:w.duration,r=arguments.length>3&&void 0!==arguments[3]&&arguments[3],i=arguments.length>4&&void 0!==arguments[4]&&arguments[4],o=function(t){return 1+--t*t*t*t*t};Ot(Vt(t=It(t,r,i)),e,o,n)}function yt(){if(0===b)return!(!Yt(0)||ct())&>(v);var t=w.friction/Math.pow(Math.abs(b),-.5);Ot(Math.pow(b,2)/t*Math.sign(b),6*Math.abs(b/t),(function(t){return 1-Math.pow(1-t,5)}))}function wt(){if(0===b)return gt(v);var t=w.friction/Math.pow(Math.abs(b),-.5),n=Math.pow(b,2)/t*Math.sign(b),e=6*Math.abs(b/t),r=(K+n)/(s/l);Ot((-1===p?Math.floor(r):Math.ceil(r))*(s/l)-K,e,(function(t){return 1-Math.pow(1-t,5)}))}function Mt(){if(ht(),0===b)return gt(v,!0);var t=.04/Math.pow(Math.abs(b),-.5),n=Math.pow(b,2)/t*Math.sign(b),e=function(t){return--t*t*t+1},r=b;Ot(n,3*Math.abs(r/t),e,!0,(function(){Ot(Vt(It(v)),500,e,!0)}))}function Ot(t,n,e,r,i){ht(),D=t,X=0,C=n,L=e,Y=r,z=i,T=null,dt()}function St(e){var r=a(t);r.length&&(n=r[0],Et(e),rt(),ot("mounted"))}function jt(){var t,n=H.breakpoints||[];for(var r in n)window.matchMedia(r).matches&&(t=r);if(t===F)return!0;var i=(F=t)?n[F]:H;i.breakpoints&&F&&delete i.breakpoints,w=e(e(e({},Ut),H),i),W=!0,h=null,kt()}function At(t){return"function"==typeof t?t():u(t,1,Math.max(ct()?o-1:o,1))}function xt(){jt(),_=!0,ot("created")}function kt(t,n){t&&(H=t),n&&(F=null),Tt(),St(n)}function Et(t){var e=window.innerWidth;if(jt()&&(e!==h||t)){h=e;var r=w.slides;"number"==typeof r?(f=null,o=r):(f=a(r,n),o=f?f.length:0);var i=w.dragSpeed*(ft()?-1:1);k="function"==typeof i?i:function(t){return t*i},s=lt()?n.offsetHeight:n.offsetWidth,l=At(w.slidesPerView),d=u(w.spacing,0,s/(l-1)-1),s+=d,c=at()?(s/2-s/l/2)/s:0,Ct();var p=!_||W&&w.resetSlide?w.initial:v;Rt(ct()?p:zt(p)),lt()&&n.setAttribute(V,!0),W=!1}}function Pt(t){Et(),setTimeout(Et,500),setTimeout(Et,2e3)}function Tt(){it(),Lt(),n&&n.hasAttribute(V)&&n.removeAttribute(V),ot("destroyed")}function Dt(){f&&f.forEach((function(t,n){var e=g[n].distance*s-n*(s/l-d/l-d/l*(l-1)),r=lt()?0:e,i=lt()?e:0,o="translate3d(".concat(r,"px, ").concat(i,"px, 0)");t.style.transform=o,t.style["-webkit-transform"]=o}))}function Ct(){f&&f.forEach((function(t){var n="calc(".concat(100/l,"% - ").concat(d/l*(l-1),"px)");lt()?(t.style["min-height"]=n,t.style["max-height"]=n):(t.style["min-width"]=n,t.style["max-width"]=n)}))}function Lt(){if(f){var t=["transform","-webkit-transform"];t=[].concat(r(t),lt?["min-height","max-height"]:["min-width","max-width"]),f.forEach((function(n){t.forEach((function(t){n.style.removeProperty(t)}))}))}}function Xt(t){var n=!(arguments.length>1&&void 0!==arguments[1])||arguments[1],e=arguments.length>2&&void 0!==arguments[2]?arguments[2]:Date.now();Ft(t,e),n&&(t=Kt(t)),K+=t,_t()}function Yt(t){var n=s*(o-1*(at()?1:l))/l,e=K+t;return e>n?e-n:e<0?e:0}function zt(t){return u(t,0,o-1-(at()?0:l-1))}function Ht(){var t=Math.abs(y),n=K<0?1-t:t;return{direction:p,progressTrack:n,progressSlides:n*o/(o-1),positions:g,position:K,speed:b,relativeSlide:(v%o+o)%o,absoluteSlide:v,size:o,slidesPerView:l,widthOrHeight:s}}function It(t){var n=arguments.length>1&&void 0!==arguments[1]&&arguments[1],e=arguments.length>2&&void 0!==arguments[2]&&arguments[2];return ct()?n?qt(t,e):t:zt(t)}function Vt(t){return-(-s/l*t+K)}function qt(t,n){var e=(v%o+o)%o,r=e<(t=(t%o+o)%o)?-e-o+t:-(e-t),i=e>t?o-e+t:t-e,a=n?Math.abs(r)<=i?r:i:t(o-1)/l?-o/l:e<-o/l+1?o/l:0);var r=1/l,i=e+r,a=i1?1-(i-1)*l/1:1;t.push({portion:a<0||a>1?0:a,distance:ft()?-1*e+1-r:e})}g=t,Dt(),ot("move")}function Kt(t){if(ct())return t;var n=Yt(t);if(!st())return t-n;if(0===n)return t;var e;return t*(e=n/s,(1-Math.abs(e))*(1-Math.abs(e)))}function Nt(){var t=Math.round(K/(s/l));t!==v&&(v=t,ot("slideChanged"))}function Rt(t){ot("beforeChange"),Xt(Vt(t),!1),ot("afterChange")}var Ut={centered:!1,breakpoints:null,controls:!0,dragSpeed:1,friction:.0025,loop:!1,initial:0,duration:500,preventEvent:"data-keen-slider-pe",slides:".keen-slider__slide",vertical:!1,resetSlide:!1,slidesPerView:1,spacing:0,mode:"snap",rtl:!1,rubberband:!0},$t={controls:function(t){i=t},destroy:Tt,refresh:function(t){kt(t,!0)},next:function(){gt(v+1,!0)},prev:function(){gt(v-1,!0)},moveToSlide:function(t,n){gt(t,!0,n)},moveToSlideRelative:function(t){var n=arguments.length>1&&void 0!==arguments[1]&&arguments[1],e=arguments.length>2?arguments[2]:void 0;gt(t,!0,e,!0,n)},resize:function(){Et(!0)},details:function(){return Ht()}};return xt(),$t}})); +//# sourceMappingURL=keen-slider.js.map \ No newline at end of file diff --git a/website/www/site/static/js/quotes-slider.js b/website/www/site/static/js/quotes-slider.js new file mode 100644 index 000000000000..27f2d9313083 --- /dev/null +++ b/website/www/site/static/js/quotes-slider.js @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 (the 'License'); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +var slider = new KeenSlider("#my-keen-slider", { + loop: true, + created: function (instance) { + var dots_wrapper = document.getElementById("dots"); + var slides = document.querySelectorAll(".keen-slider__slide"); + slides.forEach(function (t, idx) { + var dot = document.createElement("button"); + dot.classList.add("dot"); + dots_wrapper.appendChild(dot); + dot.addEventListener("click", function () { + instance.moveToSlide(idx); + }); + }); + updateClasses(instance); + }, + slideChanged(instance) { + updateClasses(instance); + } +}); + +function updateClasses(instance) { + var slide = instance.details().relativeSlide; + + var dots = document.querySelectorAll(".dot"); + dots.forEach(function (dot, idx) { + idx === slide + ? dot.classList.add("dot--active") + : dot.classList.remove("dot--active"); + }); +} \ No newline at end of file diff --git a/website/www/site/static/js/section-nav.js b/website/www/site/static/js/section-nav.js index 50d7a3f58dbc..1d40720f2202 100644 --- a/website/www/site/static/js/section-nav.js +++ b/website/www/site/static/js/section-nav.js @@ -24,7 +24,7 @@ $(document).ready(function () { var CONST = { ACTIVE_CLASS: "active", EXPANDED_CLASS: "expanded", - DESKTOP_BREAKPOINT: 1024, + DESKTOP_BREAKPOINT: 1280, OPEN_CLASS: "open" }; From 14e0a70ea69bd1ff328e4e504d3119053b254e28 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 28 Dec 2020 16:04:21 +0100 Subject: [PATCH 02/18] minor fixes --- website/www/site/assets/scss/_navbar-desktop.scss | 6 +++--- website/www/site/assets/scss/_navbar-mobile.sass | 6 +----- website/www/site/layouts/shortcodes/table_with_icons.html | 8 ++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/website/www/site/assets/scss/_navbar-desktop.scss b/website/www/site/assets/scss/_navbar-desktop.scss index 2a7e2564467d..b9a40a3007d2 100644 --- a/website/www/site/assets/scss/_navbar-desktop.scss +++ b/website/www/site/assets/scss/_navbar-desktop.scss @@ -163,12 +163,12 @@ .navigation-bar-desktop { display: none; } - + .navigation-bar-mobile { display: block; width: 100%; height: 64px; - padding: 9px 25.4px 9px 13px; + padding: 9px 25px 9px 13px; box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); background-color: $color-white; z-index: 10000; @@ -177,4 +177,4 @@ .page-nav { margin-top: 30px; } -} \ No newline at end of file +} diff --git a/website/www/site/assets/scss/_navbar-mobile.sass b/website/www/site/assets/scss/_navbar-mobile.sass index f216aa723f1d..f7a4d85ca6ef 100644 --- a/website/www/site/assets/scss/_navbar-mobile.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -25,11 +25,7 @@ text-transform: uppercase .navbar-link font-size: 16px; - font-weight: normal; - font-stretch: normal; - font-style: normal; - line-height: normal; - letter-spacing: normal + .navbar-header float: none display: flex diff --git a/website/www/site/layouts/shortcodes/table_with_icons.html b/website/www/site/layouts/shortcodes/table_with_icons.html index 4ac8dcaf221f..98a6b16d3f4f 100644 --- a/website/www/site/layouts/shortcodes/table_with_icons.html +++ b/website/www/site/layouts/shortcodes/table_with_icons.html @@ -25,11 +25,11 @@ {{ with resources.Get .icon }} {{ .Content | safeHTML }} {{ end }} -

    {{.purpose | safeHTML}}

    +

    {{ .purpose | safeHTML }}

    - {{.contact.email}} + {{.contact.email }} {{ if .contact.add }} {{ .contact.add }} {{end}} @@ -38,7 +38,7 @@
    {{ range .actions }} {{ if .link }} - {{.action |safeHTML }} + {{ .action |safeHTML }} {{ else }} {{ .action | safeHTML }} {{end}} @@ -48,4 +48,4 @@ {{ end }} -
    \ No newline at end of file +
    From 8c8c46e81264f2519df86ca31bc21c5f8cbb65a4 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 28 Dec 2020 17:09:32 +0100 Subject: [PATCH 03/18] fix mobile navbar --- website/www/site/assets/scss/_navbar-mobile.sass | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/www/site/assets/scss/_navbar-mobile.sass b/website/www/site/assets/scss/_navbar-mobile.sass index f7a4d85ca6ef..7e1450c09d83 100644 --- a/website/www/site/assets/scss/_navbar-mobile.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -17,7 +17,7 @@ .navbar padding: 0 !important - :before + ::before display: none ::after display: none @@ -39,6 +39,7 @@ align-items: center +md margin-right: $pad + margin-left: 10px .navbar-right margin-right: $pad From af9547faa1d75fcccdbaf7104527cb49ee837710 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 4 Jan 2021 15:51:08 +0100 Subject: [PATCH 04/18] delete br's --- website/www/site/assets/scss/_typography.scss | 68 +++++++++++++++---- .../www/site/content/en/community/_index.md | 4 +- .../site/content/en/community/contact-us.md | 17 +---- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index fc3c21adcdbb..fe16ffc83dc7 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -135,26 +135,68 @@ } } -h1{ - @media (max-width: $tablet){ +h1 { + @media (max-width: $tablet) { font-size: 32px; line-height: 1.19; font-weight: 500; + letter-spacing: normal; } - font-size: 46px; - font-weight: normal; - line-height: 1; + font-size: 36px; + font-weight: 500; + line-height: 1.1; + letter-spacing: normal; + margin-bottom: 24px; + margin-top: 64px; } -h2{ - @media (max-width: $tablet){ - font-size: 24px; - line-height: 1.25; +h2 { + @media (max-width: $tablet) { + font-size: 28px; + line-height: 1.29; font-weight: 500; + letter-spacing: normal; } - font-size: 29px; - line-height: 1.24; + font-size: 30px; + font-weight: 500; + line-height: 1.2; + letter-spacing: normal; + margin-bottom: 24px; + margin-top: 56px; +} +h3 { + font-size: 24px; + font-weight: 500; + line-height: 1.25; + letter-spacing: normal; + margin-bottom: 16px; + margin-top: 48px; } -p{ +h4 { + font-size: 22px; + font-weight: 500; + line-height: 1.36; + letter-spacing: normal; + margin-bottom: 12px; + margin-top: 48px; +} +h5 { + font-size: 18px; + font-weight: 500; + line-height: 1.6; + letter-spacing: 0.43px; + margin-bottom: 12px; + margin-top: 32px; +} +h6 { + font-size: 16px; + font-weight: bold; + line-height: 1.2; + letter-spacing: 0.43px; + margin-bottom: 12px; + margin-top: 32px; +} + +p { font-size: 16px; font-weight: normal; line-height: 1.63; @@ -162,4 +204,4 @@ p{ } .hero-heading { font-size: 32px; -} \ No newline at end of file +} diff --git a/website/www/site/content/en/community/_index.md b/website/www/site/content/en/community/_index.md index 11e6bb4848eb..b92ac8118a70 100644 --- a/website/www/site/content/en/community/_index.md +++ b/website/www/site/content/en/community/_index.md @@ -5,6 +5,7 @@ aliases: - /use/mailing-lists/ - /get-started/support/ --- + # Welcome to the Beam Community! -
    Beam is a tool created by community for community. We tirelessly work to make it better and you can do it too! If you are a data or software developer this is a place for you. -{{< list_with_icons >}} \ No newline at end of file +{{< list_with_icons >}} diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index 7da187e6526b..d0888c711977 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -6,6 +6,7 @@ aliases: - /use/mailing-lists/ - /get-started/support/ --- + - # Contact us! -
    - There are many ways to reach the Beam user and developer communities - use whichever one seems best. -
    -
    - ## How to use mailing lists?/What is it? -Here should be a short description on what mailing lists are and how to use them. Also information on how does subscribe, unsubscribe and archives do work (blank email etc). - -
    +Here should be a short description on what mailing lists are and how to use them. Also information on how does subscribe, unsubscribe and archives do work (blank email etc). ## Available points of contact + Choose the mailing list that suits you best and send us an email!
    -
    - {{< table_with_icons >}} -
    - If you have questions about how to use Apache Beam, we recommend you try out the [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list, and [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam). If you wish to report a security vulnerability, please contact [security@apache.org](mailto:security@apache.org). Apache Beam follows the typical [Apache vulnerability handling process](https://apache.org/security/committers.html#vulnerability-handling). - From 117815f5001e4cde01fe70d276ccd33a17e3a989 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 4 Jan 2021 15:56:32 +0100 Subject: [PATCH 05/18] delete br --- .../www/site/assets/scss/_table-wrapper.scss | 71 ++++++++++--------- .../site/content/en/community/contact-us.md | 1 - 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/website/www/site/assets/scss/_table-wrapper.scss b/website/www/site/assets/scss/_table-wrapper.scss index 226c331e1fc6..d854d5c2201d 100644 --- a/website/www/site/assets/scss/_table-wrapper.scss +++ b/website/www/site/assets/scss/_table-wrapper.scss @@ -14,90 +14,93 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - $padding:24px; -.table-wrapper{ - table{ +$padding: 24px; +.table-wrapper { + margin-top: 64px; + + table { @extend .table; max-width: 853px; } th:nth-child(1) { - @media (max-width: $mobile){ - padding-left:0 !important; + @media (max-width: $mobile) { + padding-left: 0 !important; width: 115px; - } - padding-left: 67px+$padding !important; - width: 491px; + } + padding-left: 67px + $padding !important; + width: 491px; } - th:nth-child(2) { - @media (max-width: $mobile){ + th:nth-child(2) { + @media (max-width: $mobile) { width: 111px; } - width: 181px; + width: 181px; } - th{ + th { padding: 14px 14px 17px 14px !important; border: none !important; border-bottom: 1px solid #ff6d05 !important; letter-spacing: 2px; line-height: normal !important; } - td{ - @media (max-width: $mobile){ + td { + @media (max-width: $mobile) { padding: $padding/4 !important; } padding: $padding !important; border-bottom: 1px solid rgba(255, 109, 5, 0.24); - div{ + div { display: flex; position: relative; } - img, svg{ - @media (max-width: $mobile){ + img, + svg { + @media (max-width: $mobile) { display: none; } position: absolute; - top:50%; + top: 50%; transform: translateY(-60%); } - p{ - @media (max-width: $mobile){ - margin:0; + p { + @media (max-width: $mobile) { + margin: 0; } margin-left: 67px; font-size: 14px; line-height: 1.57; } } - td:nth-child(3) { - @media (max-width: $mobile){ + td:nth-child(3) { + @media (max-width: $mobile) { padding: $padding/3 !important; padding-left: $padding/4 !important; } padding: $padding/3 !important; padding-left: $padding !important; - div{ + div { flex-direction: column; } - a{ + a { text-decoration: underline; line-height: 2; letter-spacing: 0.43px; } } - td:nth-child(2) { + td:nth-child(2) { background-color: rgba(255, 109, 0, 0.04); padding-left: $padding/2 !important; padding-right: 0 !important; line-height: 1.57; letter-spacing: 0.43px; } - a{ - color:#f26628; + a { + color: #f26628; + } +} + +.table-bordered-wrapper { + table { + @extend .table-bordered; } } - -.table-bordered-wrapper{ - table{ - @extend .table-bordered - } -} \ No newline at end of file diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index d0888c711977..e97ce6bfae89 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -34,7 +34,6 @@ Here should be a short description on what mailing lists are and how to use them Choose the mailing list that suits you best and send us an email! -
    {{< table_with_icons >}} If you have questions about how to use Apache Beam, we recommend you try out the [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list, and [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam). From a1f54273ff02cd70a9d077a2db3ae1f6349c67ac Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Tue, 5 Jan 2021 13:50:44 +0100 Subject: [PATCH 06/18] change header size --- website/www/site/layouts/shortcodes/list_with_icons.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/www/site/layouts/shortcodes/list_with_icons.html b/website/www/site/layouts/shortcodes/list_with_icons.html index c5f3b63a6401..df8b0c9f036c 100644 --- a/website/www/site/layouts/shortcodes/list_with_icons.html +++ b/website/www/site/layouts/shortcodes/list_with_icons.html @@ -9,7 +9,7 @@ See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. */}} - + {{ $data := index $.Site.Data .Site.Language.Lang }}
    {{ range $item := $data.community_list }} @@ -20,10 +20,10 @@ {{ end }}
    -
    {{ .title | markdownify }}
    +

    {{ .title | markdownify }}

    {{ .body | markdownify }}

    - + {{ end }} -
    \ No newline at end of file + From e9fc21b2c5dd60de5d03de1a06e78188d9173e0f Mon Sep 17 00:00:00 2001 From: Jakub-Sadowski Date: Sun, 10 Jan 2021 17:36:22 +0100 Subject: [PATCH 07/18] Update website/www/site/data/en/community_list.yaml Co-authored-by: Brian Hulette --- website/www/site/data/en/community_list.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/data/en/community_list.yaml b/website/www/site/data/en/community_list.yaml index 2f667b944b63..38df1c18ad79 100644 --- a/website/www/site/data/en/community_list.yaml +++ b/website/www/site/data/en/community_list.yaml @@ -14,7 +14,7 @@ body: Here you will find real life cases. icon: icons/community/bee-icon.svg - title: Contact us! - body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of them info all of the info on who to contact. + body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of the info on who to contact. icon: icons/community/envelope-icon.svg - title: Keep up with Beam body: Check out our social media and be first to know about any updates. From 272cc9d2d0d9b2bf855eac1bbf30930939b495ef Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Sun, 17 Jan 2021 16:45:23 +0100 Subject: [PATCH 08/18] add content, change shortcode --- .../www/site/content/en/community/_index.md | 2 +- .../site/content/en/community/contact-us.md | 25 +++++++++------ .../layouts/shortcodes/list_with_icons.html | 32 +++++++++---------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/website/www/site/content/en/community/_index.md b/website/www/site/content/en/community/_index.md index b92ac8118a70..0a17d98f7567 100644 --- a/website/www/site/content/en/community/_index.md +++ b/website/www/site/content/en/community/_index.md @@ -25,4 +25,4 @@ limitations under the License. Beam is a tool created by community for community. We tirelessly work to make it better and you can do it too! If you are a data or software developer this is a place for you. -{{< list_with_icons >}} +{{< list_with_icons community_list >}} diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index e97ce6bfae89..37cc44eb5604 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -23,19 +23,26 @@ limitations under the License. # Contact us! -There are many ways to reach the Beam user and developer communities - use -whichever one seems best. +The Apache Beam community is friendly and welcoming. We are glad to help with any +question, suggestion or idea you have. Contact us in the following channels: -## How to use mailing lists?/What is it? +### Available contact channels -Here should be a short description on what mailing lists are and how to use them. Also information on how does subscribe, unsubscribe and archives do work (blank email etc). +{{< table_with_icons >}} -## Available points of contact +### Mailing list, what are they and how they work -Choose the mailing list that suits you best and send us an email! +The official communication channels for Apache projects are their mailing lists, and Apache +Beam has two main lists, user@beam.apache.org and dev@beam.apache.org. You can see +what are the topics for each of the above. -{{< table_with_icons >}} +### Subsribe and Unsubscribe: + +To send emails to these lists you need to subscribe first. To subscribe send a blank email to [user@](user-subscribe@beam.apache.org) or [dev@](dev@beam.apache.org) depending on the list you want to write to. -If you have questions about how to use Apache Beam, we recommend you try out the [user@](https://lists.apache.org/list.html?user@beam.apache.org) mailing list, and [StackOverflow](https://stackoverflow.com/questions/tagged/apache-beam). +### Useful Tips for Sending Emails -If you wish to report a security vulnerability, please contact [security@apache.org](mailto:security@apache.org). Apache Beam follows the typical [Apache vulnerability handling process](https://apache.org/security/committers.html#vulnerability-handling). +- Tip1: Use tags in your subject line + A tag is a word within a pair of brackets [] that indicate the type of message you’re sending. For example: [Bug] or [Proposal] or [Question] or [Idea]. Tags help folks navigate emails easier. +- Tip 2: If you’re asking a troubleshooting question, provide as much information as possible to help others replicate your issue or find possible solutions. +- Tip 3: Share complete links instead of hyperlinks. A common practice in Apache is adding a number tag like [1] to indicate a word is a link or an attachment, and use the tag as a footnote to add the complete link at the end of your message. diff --git a/website/www/site/layouts/shortcodes/list_with_icons.html b/website/www/site/layouts/shortcodes/list_with_icons.html index df8b0c9f036c..45240b29dbdf 100644 --- a/website/www/site/layouts/shortcodes/list_with_icons.html +++ b/website/www/site/layouts/shortcodes/list_with_icons.html @@ -10,20 +10,20 @@ limitations under the License. See accompanying LICENSE file. */}} - {{ $data := index $.Site.Data .Site.Language.Lang }} -
    - {{ range $item := $data.community_list }} -
    -
    - {{ with resources.Get .icon }} - {{ .Content | safeHTML }} - {{ end }} -
    -
    -

    {{ .title | markdownify }}

    -

    {{ .body | markdownify }}

    -
    -
    + {{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} +
    + {{ range $item := $data }} +
    +
    + {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +
    +
    +

    {{ .title | markdownify }}

    +

    {{ .body | markdownify }}

    +
    +
    - {{ end }} -
    + {{ end }} +
    From b3cdce18a2b7ef90fad7b25f39548c3b9672740b Mon Sep 17 00:00:00 2001 From: Jakub-Sadowski Date: Fri, 22 Jan 2021 10:13:06 +0100 Subject: [PATCH 09/18] Update website/www/site/content/en/community/contact-us.md Co-authored-by: Brian Hulette --- website/www/site/content/en/community/contact-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index 37cc44eb5604..879157f8865d 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -38,7 +38,7 @@ what are the topics for each of the above. ### Subsribe and Unsubscribe: -To send emails to these lists you need to subscribe first. To subscribe send a blank email to [user@](user-subscribe@beam.apache.org) or [dev@](dev@beam.apache.org) depending on the list you want to write to. +To send emails to these lists you need to be subscribed. To subscribe send a blank email to [user-subscribe@beam.apache.org](user-subscribe@beam.apache.org) or [dev-subscribe@beam.apache.org](dev-subscribe@beam.apache.org) depending on the list you want to write to. ### Useful Tips for Sending Emails From 178d1a58d064ed4bd7a5aff6debf4e381995e07c Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Thu, 28 Jan 2021 13:28:02 +0100 Subject: [PATCH 10/18] fixes/changes without content update --- .../site/assets/scss/_list-with-icons.scss | 43 --- website/www/site/assets/scss/_lists.scss | 295 ++++++++++++++++++ .../www/site/assets/scss/_navbar-desktop.scss | 48 ++- .../www/site/assets/scss/_navbar-mobile.sass | 40 ++- .../www/site/assets/scss/_table-wrapper.scss | 1 + website/www/site/assets/scss/_typography.scss | 20 +- website/www/site/assets/scss/main.scss | 4 +- .../www/site/content/en/community/_index.md | 2 +- .../site/content/en/community/contact-us.md | 2 +- website/www/site/data/en/community_list.yaml | 8 +- website/www/site/data/en/contact_us.yaml | 140 +++++++-- website/www/site/i18n/navbar/en.yaml | 4 +- website/www/site/layouts/partials/header.html | 231 +++++++++----- .../{ => community}/list_with_icons.html | 4 +- .../{ => community}/table_with_icons.html | 17 +- website/www/site/static/js/section-nav.js | 33 +- 16 files changed, 689 insertions(+), 203 deletions(-) delete mode 100644 website/www/site/assets/scss/_list-with-icons.scss create mode 100644 website/www/site/assets/scss/_lists.scss rename website/www/site/layouts/shortcodes/{ => community}/list_with_icons.html (89%) rename website/www/site/layouts/shortcodes/{ => community}/table_with_icons.html (79%) diff --git a/website/www/site/assets/scss/_list-with-icons.scss b/website/www/site/assets/scss/_list-with-icons.scss deleted file mode 100644 index fc4880efd295..000000000000 --- a/website/www/site/assets/scss/_list-with-icons.scss +++ /dev/null @@ -1,43 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - .icon-list{ - margin-top: 64px; - max-width: 640px; - .list-item{ - display:flex; - margin-bottom: 31px; - - .list-item-icon{ - width: 44px; - height: 44px; - display: block; - margin-right: 25px; - } - .list-item-header{ - font-size: 22px; - font-weight: normal; - letter-spacing: normal; - line-height: 1.36; - @media (max-width: $mobile){ - font-weight: 500; - } - } - } - } - \ No newline at end of file diff --git a/website/www/site/assets/scss/_lists.scss b/website/www/site/assets/scss/_lists.scss new file mode 100644 index 000000000000..392b476b7892 --- /dev/null +++ b/website/www/site/assets/scss/_lists.scss @@ -0,0 +1,295 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import "media"; + +.arrow-list { + margin-top: 26px; + h3 { + max-width: 555px; + } + ul { + font-size: 16px; + font-weight: normal; + line-height: 1.63; + letter-spacing: 0.43px; + position: relative; + padding-left: 26px; + li::marker { + content: "\2192"; + color: #ff6d05; + font-size: 24px; + } + li { + list-style-type: none; + list-style-position: outside; + padding-left: 8px; + margin-bottom: -2px; + } + li:last-of-type { + margin-bottom: 25px; + } + } + figure { + width: 112px; + height: 112px; + margin-top: 64px; + } + a { + color: #333333; + } +} + +.arrow-list-header { + color: #333333; + text-decoration: none; + display: flex; + max-height: 40px; + margin-bottom: 74px; + &:hover { + cursor: default; + text-decoration: none; + } + @media (max-width: $mobile) { + margin-bottom: 50px; + &:hover { + cursor: pointer; + text-decoration: underline; + } + h2 { + font-size: 24px; + line-height: 1.25; + font-weight: 500; + letter-spacing: normal; + margin-top: 46px; + } + } + figure { + margin-top: 40px; + margin-right: 5px; + width: auto; + height: auto; + @media (min-width: $mobile) { + display: none; + } + img { + width: 18px; + height: 18px; + padding-right: 16px; + transform: rotate(-90deg); + } + } +} +.rotate { + img { + transform: rotate(0deg) !important; + margin-top: 10px; + } +} +.list { + margin-top: 32px; + max-width: 640px; +} +.icon-list { + .list-item { + display: flex; + margin-bottom: 38px; + position: relative; + + img { + position: absolute; + top: 7px; + z-index: 2; + } + .reverse { + transform: rotate(-180deg); + } + + .cirkle { + width: 24px; + height: 24px; + background: #ff6d05; + border-radius: 50%; + text-align: center; + z-index: 5; + position: relative; + i { + vertical-align: bottom; + color: #fff; + } + } + .list-item-icon { + width: 44px; + height: 44px; + display: block; + margin-right: 25px; + } + .list-item-header { + font-size: 22px; + font-weight: 500; + letter-spacing: normal; + line-height: 1.36; + margin-top: 10px; + a { + color: #333333; + } + @media (max-width: $mobile) { + font-weight: 500; + } + } + .list-item-description { + @media (max-width: $mobile) { + position: absolute; + left: 55px; + } + } + } + .pillars-item { + display: flex; + align-items: center; + margin-bottom: 14px; + .pillars-item-icon { + margin-right: 28px; + } + + .pillars-item-description { + width: 100%; + max-width: 684px; + + .pillars-item-header { + font-size: 16px; + font-weight: bold; + line-height: 1.2; + letter-spacing: 0.43px; + margin-bottom: 6px; + margin-top: 10px; + } + + .pillars-item-text { + font-size: 14px; + font-weight: normal; + line-height: 1.57; + letter-spacing: 0.43px; + } + } + } +} +.documentation-list { + padding-left: 12px; + display: flex; + flex-direction: row; + justify-content: space-between; + flex-wrap: wrap; + max-width: 895px; + .row { + display: flex; + flex-direction: column; + margin-top: 63px; + @media (max-width: $mobile) { + max-width: 260px; + } + + .item-icon { + height: 93px; + max-width: 260px; + display: flex; + align-items: center; + @media (max-width: $mobile) { + justify-content: center; + } + } + a { + font-size: 16px; + font-weight: bold; + line-height: 1.63; + letter-spacing: 0.43px; + color: #333333; + } + .item-description { + max-width: 260px; + margin-right: 32px; + margin-top: 20px; + @media (max-width: $mobile) { + margin-right: 0; + width: auto; + } + } + } +} +.sdks { + max-width: 635px; + .column { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + } + .row { + margin-top: 48px; + .item-description { + margin-right: 0; + } + } +} +.collapsable-list { + margin: 52px 0; + a { + font-size: 14px; + font-weight: bold; + font-stretch: normal; + font-style: normal; + line-height: 1.14; + letter-spacing: 0.6px; + color: #f26628; + } + li { + a { + font-size: 16px; + line-height: 1.63; + letter-spacing: 0.43px; + color: #333333; + } + } +} +.mobile-column { + @media (max-width: $mobile) { + flex-direction: column; + align-items: center; + text-align: center; + } +} +.margin-50 { + margin-top: 50px; +} +@media (min-width: 768px) { + .arrow-list { + .collapse.dont-collapse-sm { + display: block; + height: auto !important; + visibility: visible; + } + .collapse { + display: block; + height: auto !important; + visibility: visible; + } + .collapsing { + position: relative; + height: unset !important; + overflow: hidden; + } + } +} diff --git a/website/www/site/assets/scss/_navbar-desktop.scss b/website/www/site/assets/scss/_navbar-desktop.scss index b9a40a3007d2..d3eb4e9832f3 100644 --- a/website/www/site/assets/scss/_navbar-desktop.scss +++ b/website/www/site/assets/scss/_navbar-desktop.scss @@ -19,43 +19,57 @@ .navigation-bar-mobile { display: none; + .arrow-icon { + display: flex; + align-items: baseline; + margin-left: 10px; + } +} +.nav-tabs { + border-bottom: none; } - .navigation-bar-desktop { display: flex; height: 96px; width: 100%; align-items: center; - justify-content: space-between; margin-bottom: 30px; box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); background-color: $color-white; z-index: 10000; // just to make sure that navbar always on top of other elements + #iconsBar { + display: flex; + } + a { @extend .component-text; - + font-weight: 500; color: $color-dark-gray; - letter-spacing: 0.2; - margin-right: 56px; + letter-spacing: 0.2px; + line-height: 1.63; + margin-right: 39px; text-decoration: none; cursor: pointer; } .navbar-logo { margin-left: 58px; - + margin-bottom: 4px; img { width: 88px; } } - + .navbar-bar-left { + display: flex; + justify-content: space-between; + width: 100%; + } .navbar-links { display: flex; align-items: center; justify-content: space-between; z-index: 10000; - :last-child { margin-right: 0; } @@ -96,9 +110,14 @@ box-shadow: none; padding-top: 34px; padding-bottom: 0; - + border-radius: 0; + li { + height: 36px; + } a { @extend .component-text; + line-height: 1.63; + letter-spacing: 0.2px; } } } @@ -121,7 +140,10 @@ box-shadow: none; padding-top: 35px; padding-bottom: 0; - + border-radius: 0; + li { + height: 36px; + } a { @extend .component-text; @@ -166,12 +188,6 @@ .navigation-bar-mobile { display: block; - width: 100%; - height: 64px; - padding: 9px 25px 9px 13px; - box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); - background-color: $color-white; - z-index: 10000; } .page-nav { diff --git a/website/www/site/assets/scss/_navbar-mobile.sass b/website/www/site/assets/scss/_navbar-mobile.sass index 7e1450c09d83..4c9a2a528808 100644 --- a/website/www/site/assets/scss/_navbar-mobile.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -17,6 +17,7 @@ .navbar padding: 0 !important + min-height: 64px ::before display: none ::after @@ -25,13 +26,16 @@ text-transform: uppercase .navbar-link font-size: 16px; - + font-weight: 500; + line-height: normal; + letter-spacing: normal .navbar-header float: none display: flex align-items: center justify-content: space-between height: 100% + margin-top: 5px .navbar-brand padding: 0 @@ -39,7 +43,6 @@ align-items: center +md margin-right: $pad - margin-left: 10px .navbar-right margin-right: $pad @@ -68,16 +71,38 @@ background-color: $color-white bottom: 0 min-height: 100vh - max-width: 256px + max-width: 303px padding: 15px position: fixed top: 0 transition: transform 100ms linear - width: calc(100% - 32px) + width: calc(100% - 72px) right: 0 - + overflow-y: auto + + .navbar-toggle + margin: 0 + .dropdown-toggle + display: flex + align-items: center + .navbar-nav + margin-top: 58px .navbar-nav > li width: 100% + padding: 5px 0 + + span.navbar-link + padding: 10px 15px + color: #555555 + ul + list-style: none !important + li + padding: 9px 0 + a + letter-spacing: 0.2px + + .navbar-link + text-transform: none &.closed transform: translateX(100%) @@ -104,7 +129,12 @@ &.open opacity: 0.5 width: 100% + overflow-y: auto @media (max-width: $tablet) .navbar-right margin-right: -15px + margin-top: 0 !important + +.fixedPosition + position: fixed diff --git a/website/www/site/assets/scss/_table-wrapper.scss b/website/www/site/assets/scss/_table-wrapper.scss index d854d5c2201d..3a7512266b81 100644 --- a/website/www/site/assets/scss/_table-wrapper.scss +++ b/website/www/site/assets/scss/_table-wrapper.scss @@ -42,6 +42,7 @@ $padding: 24px; border-bottom: 1px solid #ff6d05 !important; letter-spacing: 2px; line-height: normal !important; + text-transform: uppercase; } td { @media (max-width: $mobile) { diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index fe16ffc83dc7..a4d8877f3199 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -19,7 +19,7 @@ .component-title { font-size: 36px; - font-weight: normal; + font-weight: 500; font-style: normal; line-height: 1.1; letter-spacing: normal; @@ -129,6 +129,7 @@ @media (max-width: $tablet) { .component-title { font-size: 28px; + line-height: 1.29; } .component-large-header { font-size: 24px; @@ -136,12 +137,6 @@ } h1 { - @media (max-width: $tablet) { - font-size: 32px; - line-height: 1.19; - font-weight: 500; - letter-spacing: normal; - } font-size: 36px; font-weight: 500; line-height: 1.1; @@ -150,12 +145,6 @@ h1 { margin-top: 64px; } h2 { - @media (max-width: $tablet) { - font-size: 28px; - line-height: 1.29; - font-weight: 500; - letter-spacing: normal; - } font-size: 30px; font-weight: 500; line-height: 1.2; @@ -205,3 +194,8 @@ p { .hero-heading { font-size: 32px; } +li { + font-size: 16px; + letter-spacing: 0.43px; + line-height: 1.63; +} diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index 44806f9ac351..158aeced4449 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -43,6 +43,6 @@ @import "_page-nav.sass"; @import "_table-wrapper.scss"; @import "_calendar.scss"; -@import "list-with-icons.scss"; +@import "_lists.scss"; @import "_quotes.scss"; -@import "navbar-desktop.scss"; \ No newline at end of file +@import "navbar-desktop.scss"; diff --git a/website/www/site/content/en/community/_index.md b/website/www/site/content/en/community/_index.md index 0a17d98f7567..917bc1ef6065 100644 --- a/website/www/site/content/en/community/_index.md +++ b/website/www/site/content/en/community/_index.md @@ -25,4 +25,4 @@ limitations under the License. Beam is a tool created by community for community. We tirelessly work to make it better and you can do it too! If you are a data or software developer this is a place for you. -{{< list_with_icons community_list >}} +{{< community/list_with_icons community_list >}} diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index 37cc44eb5604..8336e38bbc42 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -28,7 +28,7 @@ question, suggestion or idea you have. Contact us in the following channels: ### Available contact channels -{{< table_with_icons >}} +{{< community/table_with_icons >}} ### Mailing list, what are they and how they work diff --git a/website/www/site/data/en/community_list.yaml b/website/www/site/data/en/community_list.yaml index 38df1c18ad79..5a303908bb35 100644 --- a/website/www/site/data/en/community_list.yaml +++ b/website/www/site/data/en/community_list.yaml @@ -10,11 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -- title: Join Beam Community +- title: Join Beam Community body: Here you will find real life cases. - icon: icons/community/bee-icon.svg -- title: Contact us! - body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of the info on who to contact. + icon: icons/community/bee-icon.svg +- title: Contact us! + body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of them info all of the info on who to contact. icon: icons/community/envelope-icon.svg - title: Keep up with Beam body: Check out our social media and be first to know about any updates. diff --git a/website/www/site/data/en/contact_us.yaml b/website/www/site/data/en/contact_us.yaml index 17af7beceac1..3ac86a424dd5 100644 --- a/website/www/site/data/en/contact_us.yaml +++ b/website/www/site/data/en/contact_us.yaml @@ -10,32 +10,114 @@ # See the License for the specific language governing permissions and # limitations under the License. -- purpose: User support and questions - contact: { email: user@, link: "https://lists.apache.org/list.html?user@beam.apache.org", add: mailing list } - icon: icons/community/contact-us/question-mark.svg - actions: [{action: Subscribe,link: "mailto:user-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:user-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?user@beam.apache.org"}] -- purpose: Development discussions - contact: { email: dev@, link: "https://lists.apache.org/list.html?dev@beam.apache.org", add: mailing list} - icon: icons/community/contact-us/discussion.svg - actions: [{action: Subscribe,link: "mailto:dev-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:dev-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?dev@beam.apache.org"}] -- purpose: Firehose of commits, bugs, pull requests, etc. - contact: { email: commits@, link: "https://lists.apache.org/list.html?commits@beam.apache.org", add: mailing list } - icon: icons/community/contact-us/gitArrows.svg - actions: [{action: Subscribe,link: "mailto:commits-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:commits-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?commits@beam.apache.org"}] -- purpose: Firehose of build notifications from Jenkins - contact: { email: builds@, link: "https://lists.apache.org/list.html?builds@beam.apache.org", add: mailing list } - icon: icons/community/contact-us/notification.svg - actions: [{action: Subscribe,link: "mailto:builds-subscribe@beam.apache.org"},{action: Unsubscribe, link: "mailto:builds-unsubscribe@beam.apache.org"}, {action: Archives, link: "https://lists.apache.org/list.html?builds@beam.apache.org"}] -- purpose: Report bugs / discover known issues - contact: { email: JIRA bug tracker, link: "https://issues.apache.org/jira/browse/BEAM" } - icon: icons/community/contact-us/bug.svg - - actions: [{action: '‐'}] -- purpose: Ask and answer user support questions - contact: { email: StackOverflow, link: "https://stackoverflow.com/questions/tagged/apache-beam" } - icon: icons/community/contact-us/messages.svg - actions: [{action: '‐'}] -- purpose: 'Chat with users and developers in the ASF Slack. Note: Please join the #beam channel after you created an account. Please do not ask Beam questions in #general.' - contact: { email: Slack, link: "https://s.apache.org/beam-slack-channel" } - icon: icons/community/contact-us/knot.svg - actions: [{action: '‐'}] +- columns: + - purpose + - contact + - action + data: + - purpose: User support and questions + contact: + { + email: user@, + link: "https://lists.apache.org/list.html?user@beam.apache.org", + add: mailing list, + } + icon: icons/community/contact-us/question-mark.svg + actions: + [ + { action: Subscribe, link: "mailto:user-subscribe@beam.apache.org" }, + { + action: Unsubscribe, + link: "mailto:user-unsubscribe@beam.apache.org", + }, + { + action: Archives, + link: "https://lists.apache.org/list.html?user@beam.apache.org", + }, + ] + - purpose: Development discussions + contact: + { + email: dev@, + link: "https://lists.apache.org/list.html?dev@beam.apache.org", + add: mailing list, + } + icon: icons/community/contact-us/discussion.svg + actions: + [ + { action: Subscribe, link: "mailto:dev-subscribe@beam.apache.org" }, + { + action: Unsubscribe, + link: "mailto:dev-unsubscribe@beam.apache.org", + }, + { + action: Archives, + link: "https://lists.apache.org/list.html?dev@beam.apache.org", + }, + ] + - purpose: Firehose of commits, bugs, pull requests, etc. + contact: + { + email: commits@, + link: "https://lists.apache.org/list.html?commits@beam.apache.org", + add: mailing list, + } + icon: icons/community/contact-us/gitArrows.svg + actions: + [ + { + action: Subscribe, + link: "mailto:commits-subscribe@beam.apache.org", + }, + { + action: Unsubscribe, + link: "mailto:commits-unsubscribe@beam.apache.org", + }, + { + action: Archives, + link: "https://lists.apache.org/list.html?commits@beam.apache.org", + }, + ] + - purpose: Firehose of build notifications from Jenkins + contact: + { + email: builds@, + link: "https://lists.apache.org/list.html?builds@beam.apache.org", + add: mailing list, + } + icon: icons/community/contact-us/notification.svg + actions: + [ + { + action: Subscribe, + link: "mailto:builds-subscribe@beam.apache.org", + }, + { + action: Unsubscribe, + link: "mailto:builds-unsubscribe@beam.apache.org", + }, + { + action: Archives, + link: "https://lists.apache.org/list.html?builds@beam.apache.org", + }, + ] + - purpose: Report bugs / discover known issues + contact: + { + email: JIRA bug tracker, + link: "https://issues.apache.org/jira/browse/BEAM", + } + icon: icons/community/contact-us/bug.svg + actions: [{ action: "‐" }] + - purpose: Ask and answer user support questions + contact: + { + email: StackOverflow, + link: "https://stackoverflow.com/questions/tagged/apache-beam", + } + icon: icons/community/contact-us/messages.svg + actions: [{ action: "‐" }] + - purpose: 'Chat with users and developers in the ASF Slack. Note: Please join the #beam channel after you created an account. Please do not ask Beam questions in #general.' + contact: { email: Slack, link: "https://s.apache.org/beam-slack-channel" } + icon: icons/community/contact-us/knot.svg + actions: [{ action: "‐" }] diff --git a/website/www/site/i18n/navbar/en.yaml b/website/www/site/i18n/navbar/en.yaml index ae6255e50683..74cc78caf1f3 100644 --- a/website/www/site/i18n/navbar/en.yaml +++ b/website/www/site/i18n/navbar/en.yaml @@ -9,7 +9,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + - id: nav-toggle-navigation translation: "Toggle navigation" - id: nav-get-started @@ -42,3 +42,5 @@ translation: "Sponsorship" - id: nav-code-of-conduct translation: "Code of Conduct" +- id: nav-powered-by + translation: "Powered by" diff --git a/website/www/site/layouts/partials/header.html b/website/www/site/layouts/partials/header.html index 2d380c649cd6..009f036f6faf 100644 --- a/website/www/site/layouts/partials/header.html +++ b/website/www/site/layouts/partials/header.html @@ -1,24 +1,25 @@ {{/* - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. -*/}} + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. + */}} - + diff --git a/website/www/site/layouts/shortcodes/list_with_icons.html b/website/www/site/layouts/shortcodes/community/list_with_icons.html similarity index 89% rename from website/www/site/layouts/shortcodes/list_with_icons.html rename to website/www/site/layouts/shortcodes/community/list_with_icons.html index 45240b29dbdf..517ab4169953 100644 --- a/website/www/site/layouts/shortcodes/list_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/list_with_icons.html @@ -19,8 +19,8 @@ {{ .Content | safeHTML }} {{ end }} -
    -

    {{ .title | markdownify }}

    +
    +

    {{ .title | safeHTML }}

    {{ .body | markdownify }}

    diff --git a/website/www/site/layouts/shortcodes/table_with_icons.html b/website/www/site/layouts/shortcodes/community/table_with_icons.html similarity index 79% rename from website/www/site/layouts/shortcodes/table_with_icons.html rename to website/www/site/layouts/shortcodes/community/table_with_icons.html index 98a6b16d3f4f..234288c8ed5d 100644 --- a/website/www/site/layouts/shortcodes/table_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/table_with_icons.html @@ -10,26 +10,26 @@ limitations under the License. See accompanying LICENSE file. */}} {{ $data := index $.Site.Data .Site.Language.Lang }} - +{{ range $list := $data.contact_us }}
    - - - + {{ range $item := $list.columns }} + + {{ end }} - {{ range $item := $data.contact_us }} + {{ range $item := $list.data }}
    PURPOSECONTACTACTION{{ . }}
    {{ with resources.Get .icon }} {{ .Content | safeHTML }} {{ end }} -

    {{ .purpose | safeHTML }}

    +

    {{.purpose | safeHTML}}

    		 - {{.contact.email }} + {{.contact.email}} {{ if .contact.add }} {{ .contact.add }} {{end}} @@ -38,7 +38,7 @@
    {{ range .actions }} {{ if .link }} - {{ .action |safeHTML }} + {{.action |safeHTML }} {{ else }} {{ .action | safeHTML }} {{end}} @@ -49,3 +49,4 @@ {{ end }}
    +{{ end }} diff --git a/website/www/site/static/js/section-nav.js b/website/www/site/static/js/section-nav.js index 1d40720f2202..841dcb338814 100644 --- a/website/www/site/static/js/section-nav.js +++ b/website/www/site/static/js/section-nav.js @@ -101,15 +101,25 @@ $(document).ready(function () { var sectionNavHeight = $(sectionNavEl).height(); var mainContent = $(".container-main-content"); - mainContent.css({"min-height": sectionNavHeight}); - sectionNavEl.css({"max-height": mainContent.css("height")}); - + //mainContent.css({"min-height": sectionNavHeight}); + //sectionNavEl.css({"max-height": mainContent.css("height")}); + $(window).load(function () { + var url = window.location.href; + var links = $("." + idSectionNav+' a'); + for(var i = 0; i < links.length; i++){ + if(links[i].classList.contains('underline')) + links[i].removeClass('underline'); + } + for(var i = 0; i < links.length; i++){ + if(url.includes(links[i].href)) + links[i].classList.add('underline'); + } + }); $(window).resize(function () { if ($(window).width() > CONST.DESKTOP_BREAKPOINT) { var sectionNavHeight = $(sectionNavEl).height(); - $(".container-main-content").css({"min-height": sectionNavHeight}); } else { - $(".container-main-content").css({"min-height": ''}); + $(".container-main-content").css({"min-height": '100vh'}); } }); @@ -133,9 +143,20 @@ $(document).ready(function () { $("." + _self.idNavMask).click(function (el) { $("." + _self.idNavMask).removeClass(CONST.OPEN_CLASS); $("." + _self.idContainer).removeClass(CONST.OPEN_CLASS); + $("body").removeClass("fixedPosition"); + if (_self.hasSectionNav) { + $("." + _self.idSectionNav).removeClass(CONST.OPEN_CLASS); + $("body").removeClass("fixedPosition"); + } + }); + $("#closeMenu").click(function (el) { + $("." + _self.idNavMask).removeClass(CONST.OPEN_CLASS); + $("." + _self.idContainer).removeClass(CONST.OPEN_CLASS); + $("body").removeClass("fixedPosition"); if (_self.hasSectionNav) { $("." + _self.idSectionNav).removeClass(CONST.OPEN_CLASS); + $("body").removeClass("fixedPosition"); } }); @@ -166,7 +187,7 @@ $(document).ready(function () { "classNameContainer": "navbar-container", "classNameSectionNav": "section-nav", "classNameBackCTA": "section-nav-back", - "classNameCTA": "navbar-toggle", + "classNameCTA": "menu-open", "classNameMask": "navbar-mask", "classNameNavItemTitleCollapsible": "section-nav-item--collapsible span", "classNameNavItemCollapsible": ".section-nav-item--collapsible", From 241cef13f40c239ae2ade295e21a1e3e454a9851 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Thu, 28 Jan 2021 13:30:13 +0100 Subject: [PATCH 11/18] merge --- website/www/site/content/en/community/contact-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index 13f665988e8f..8336e38bbc42 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -38,7 +38,7 @@ what are the topics for each of the above. ### Subsribe and Unsubscribe: -To send emails to these lists you need to be subscribed. To subscribe send a blank email to [user-subscribe@beam.apache.org](user-subscribe@beam.apache.org) or [dev-subscribe@beam.apache.org](dev-subscribe@beam.apache.org) depending on the list you want to write to. +To send emails to these lists you need to subscribe first. To subscribe send a blank email to [user@](user-subscribe@beam.apache.org) or [dev@](dev@beam.apache.org) depending on the list you want to write to. ### Useful Tips for Sending Emails From a5b75d948a39ab5541bd60ddb949e66f060d4863 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Thu, 28 Jan 2021 13:39:10 +0100 Subject: [PATCH 12/18] whitespaces --- .../www/site/layouts/community/baseof.html | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/website/www/site/layouts/community/baseof.html b/website/www/site/layouts/community/baseof.html index c84cee0bedc2..f6b0664dd3d4 100644 --- a/website/www/site/layouts/community/baseof.html +++ b/website/www/site/layouts/community/baseof.html @@ -11,30 +11,30 @@ */}} - - - {{ partial "head.html" . }} - - - {{ partial "header.html" . }} -
    -
    - - -
    - - - -
    - {{ .Content }} -
    -
    - {{ partial "footer.html" . }} - - \ No newline at end of file + + + {{ partial "head.html" . }} + + + {{ partial "header.html" . }} +
    +
    + + +
    + + + +
    + {{ .Content }} +
    +
    + {{ partial "footer.html" . }} + + From d8182d2aeb9bc9e2d2285239d493a9d23797c311 Mon Sep 17 00:00:00 2001 From: Jakub-Sadowski Date: Sat, 30 Jan 2021 21:43:17 +0100 Subject: [PATCH 13/18] Update website/www/site/static/js/section-nav.js Co-authored-by: Brian Hulette --- website/www/site/static/js/section-nav.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/www/site/static/js/section-nav.js b/website/www/site/static/js/section-nav.js index 841dcb338814..34b3d59add23 100644 --- a/website/www/site/static/js/section-nav.js +++ b/website/www/site/static/js/section-nav.js @@ -101,8 +101,6 @@ $(document).ready(function () { var sectionNavHeight = $(sectionNavEl).height(); var mainContent = $(".container-main-content"); - //mainContent.css({"min-height": sectionNavHeight}); - //sectionNavEl.css({"max-height": mainContent.css("height")}); $(window).load(function () { var url = window.location.href; var links = $("." + idSectionNav+' a'); From 849464d13ccbe17a918eda87b7cd916aeff4078d Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Sat, 30 Jan 2021 22:08:58 +0100 Subject: [PATCH 14/18] fixes --- website/www/site/assets/scss/_lists.scss | 10 --- .../www/site/assets/scss/_navbar-mobile.sass | 6 +- website/www/site/assets/scss/_search.scss | 81 +++++++++++++++++++ website/www/site/assets/scss/main.scss | 3 +- .../site/content/en/community/contact-us.md | 2 +- website/www/site/layouts/partials/header.html | 26 +++--- .../community/table_with_icons.html | 4 +- 7 files changed, 102 insertions(+), 30 deletions(-) create mode 100644 website/www/site/assets/scss/_search.scss diff --git a/website/www/site/assets/scss/_lists.scss b/website/www/site/assets/scss/_lists.scss index 392b476b7892..c2bdc03d6ef6 100644 --- a/website/www/site/assets/scss/_lists.scss +++ b/website/www/site/assets/scss/_lists.scss @@ -281,15 +281,5 @@ height: auto !important; visibility: visible; } - .collapse { - display: block; - height: auto !important; - visibility: visible; - } - .collapsing { - position: relative; - height: unset !important; - overflow: hidden; - } } } diff --git a/website/www/site/assets/scss/_navbar-mobile.sass b/website/www/site/assets/scss/_navbar-mobile.sass index 4c9a2a528808..198ced5f9443 100644 --- a/website/www/site/assets/scss/_navbar-mobile.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -25,9 +25,9 @@ .navbar-nav > li > a text-transform: uppercase .navbar-link - font-size: 16px; - font-weight: 500; - line-height: normal; + font-size: 16px + font-weight: 500 + line-height: normal letter-spacing: normal .navbar-header float: none diff --git a/website/www/site/assets/scss/_search.scss b/website/www/site/assets/scss/_search.scss new file mode 100644 index 000000000000..a035df1566eb --- /dev/null +++ b/website/www/site/assets/scss/_search.scss @@ -0,0 +1,81 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +.searchBar { + width: 297px; + margin-right: 30px; + display: flex; + align-items: center; + table { + margin: 0; + } + div { + padding: 0; + svg { + padding-top: 2px; + } + } + .gsc-search-button { + display: none; + } + #__gcse_1 { + max-width: 213px; + } + .gsib_b { + display: none; + } + .gsc-input-box { + border-radius: 100px; + background-color: #f1f1f2; + table { + margin: 0; + } + input { + background-color: #f1f1f2; + min-width: 213px; + font-size: 12px; + font-weight: 400; + line-height: 1.83; + letter-spacing: normal; + } + } +} +.disappear { + display: none !important; +} +.searchBar-mobile { + width: 100%; + table { + margin: 0; + } + .gsc-input-box { + border-radius: 100px; + max-height: 36px; + background-color: #f1f1f2; + padding-top: 2px; + input { + background-color: #f1f1f2; + transform: translateY(-2px); + } + } + .gsc-search-button { + background-color: #fff; + border: none; + svg { + fill: #ff6d00; + } + &:focus { + background-color: #fff; + } + } +} diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index 158aeced4449..b38b7e8a9736 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -45,4 +45,5 @@ @import "_calendar.scss"; @import "_lists.scss"; @import "_quotes.scss"; -@import "navbar-desktop.scss"; +@import "_navbar-desktop.scss"; +@import "_search.scss"; diff --git a/website/www/site/content/en/community/contact-us.md b/website/www/site/content/en/community/contact-us.md index 8336e38bbc42..ee22601a2664 100644 --- a/website/www/site/content/en/community/contact-us.md +++ b/website/www/site/content/en/community/contact-us.md @@ -28,7 +28,7 @@ question, suggestion or idea you have. Contact us in the following channels: ### Available contact channels -{{< community/table_with_icons >}} +{{< community/table_with_icons contact_us >}} ### Mailing list, what are they and how they work diff --git a/website/www/site/layouts/partials/header.html b/website/www/site/layouts/partials/header.html index 009f036f6faf..aff72e8fdc3e 100644 --- a/website/www/site/layouts/partials/header.html +++ b/website/www/site/layouts/partials/header.html @@ -102,12 +102,12 @@ @@ -150,7 +150,7 @@ - + {{ with resources.Get "icons/edit-icon.svg" }} {{ .Content | safeHTML }} {{ end }} @@ -166,12 +166,12 @@ diff --git a/website/www/site/layouts/shortcodes/community/table_with_icons.html b/website/www/site/layouts/shortcodes/community/table_with_icons.html index 234288c8ed5d..e97aaa5f8d87 100644 --- a/website/www/site/layouts/shortcodes/community/table_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/table_with_icons.html @@ -9,8 +9,8 @@ See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. */}} -{{ $data := index $.Site.Data .Site.Language.Lang }} -{{ range $list := $data.contact_us }} +{{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} +{{ range $list := $data }}
    From 0e45184d02530097c22552d91fd77b87036106f4 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Sat, 30 Jan 2021 22:23:22 +0100 Subject: [PATCH 15/18] add attribute for link --- website/www/site/data/en/community_list.yaml | 10 +++++----- .../layouts/shortcodes/community/list_with_icons.html | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/website/www/site/data/en/community_list.yaml b/website/www/site/data/en/community_list.yaml index 5a303908bb35..665b72672e75 100644 --- a/website/www/site/data/en/community_list.yaml +++ b/website/www/site/data/en/community_list.yaml @@ -10,18 +10,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -- title: Join Beam Community +- title: { text: Join Beam Community, link: /community/join-beam } body: Here you will find real life cases. icon: icons/community/bee-icon.svg -- title: Contact us! +- title: { text: Contact us!, link: /community/contact-us } body: You want to know what is going on? Raise a bug? Propose a feature request? Here you will find all of them info all of the info on who to contact. icon: icons/community/envelope-icon.svg -- title: Keep up with Beam +- title: { text: Keep up with Beam, link: / } body: Check out our social media and be first to know about any updates. icon: icons/community/message-icon.svg -- title: Meet up with us! +- title: { text: Meet up with us!, link: / } body: Want to meet us in person? (or monitor-person). This is the place to go. icon: icons/community/calendar-icon.svg -- title: Promotion +- title: { text: Promotion, link: / } body: Need some marketing materials? Here you will find it all. icon: icons/community/box-icon.svg diff --git a/website/www/site/layouts/shortcodes/community/list_with_icons.html b/website/www/site/layouts/shortcodes/community/list_with_icons.html index 517ab4169953..a3414d120cd8 100644 --- a/website/www/site/layouts/shortcodes/community/list_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/list_with_icons.html @@ -20,7 +20,8 @@ {{ end }}
    -

    {{ .title | safeHTML }}

    +

    + {{ .title.text }}

    {{ .body | markdownify }}

    From 3eae5fec8e13cfa4aa91cf3e6ee220a27d482ddf Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Sat, 30 Jan 2021 23:00:54 +0100 Subject: [PATCH 16/18] whitespace --- .../shortcodes/community/list_with_icons.html | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/website/www/site/layouts/shortcodes/community/list_with_icons.html b/website/www/site/layouts/shortcodes/community/list_with_icons.html index a3414d120cd8..1ada35965a69 100644 --- a/website/www/site/layouts/shortcodes/community/list_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/list_with_icons.html @@ -10,21 +10,20 @@ limitations under the License. See accompanying LICENSE file. */}} - {{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} -
    - {{ range $item := $data }} -
    -
    - {{ with resources.Get .icon }} - {{ .Content | safeHTML }} - {{ end }} -
    -
    -

    - {{ .title.text }} -

    {{ .body | markdownify }}

    -

    -
    - - {{ end }} -
    +{{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} +
    + {{ range $item := $data }} +
    +
    + {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} +
    +
    +

    + {{ .title.text }} +

    {{ .body | markdownify }}

    +

    +
    + {{ end }} +
    From cde7e34128e55eaffb4bd2b87ff50e331b401441 Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 1 Feb 2021 11:58:56 +0100 Subject: [PATCH 17/18] content update --- .../www/site/content/en/blog/beam-2.13.0.md | 2 +- .../www/site/content/en/blog/beam-2.21.0.md | 2 +- .../www/site/content/en/blog/beam-2.22.0.md | 2 +- .../www/site/content/en/blog/beam-2.23.0.md | 4 +- .../www/site/content/en/blog/beam-2.24.0.md | 4 +- .../www/site/content/en/blog/beam-2.25.0.md | 4 +- .../content/en/blog/pattern-match-beam-sql.md | 2 +- .../www/site/content/en/contribute/_index.md | 2 +- .../content/en/contribute/jira-priorities.md | 1 + .../content/en/contribute/release-guide.md | 11 +- .../sql/extensions/create-external-table.md | 15 +- .../en/documentation/dsls/sql/overview.md | 2 +- .../io/built-in/google-bigquery.md | 17 +- .../en/documentation/io/built-in/snowflake.md | 6 +- .../en/documentation/programming-guide.md | 154 ++++++++- .../en/documentation/runners/direct.md | 112 ++----- .../content/en/documentation/runners/flink.md | 42 ++- .../content/en/documentation/runners/samza.md | 5 + .../en/documentation/runtime/environments.md | 305 ++++++++++++------ .../content/en/get-started/beam-overview.md | 29 +- .../site/content/en/get-started/downloads.md | 30 +- .../content/en/get-started/quickstart-java.md | 24 +- 22 files changed, 520 insertions(+), 255 deletions(-) diff --git a/website/www/site/content/en/blog/beam-2.13.0.md b/website/www/site/content/en/blog/beam-2.13.0.md index 2f9f85595551..ac62b63b273d 100644 --- a/website/www/site/content/en/blog/beam-2.13.0.md +++ b/website/www/site/content/en/blog/beam-2.13.0.md @@ -7,7 +7,7 @@ categories: aliases: - /blog/2019/05/22/beam-2.13.0.html authors: - - goenka + - angoenka --- We are happy to present the new 2.21.0 release of Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2210-2020-05-27) for this release. +See the [download page](/get-started/downloads/#2210-2020-05-27) for this release. For more information on changes in 2.21.0, check out the [detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347143). diff --git a/website/www/site/content/en/blog/beam-2.22.0.md b/website/www/site/content/en/blog/beam-2.22.0.md index a468fb6f1e02..17fe4d27a0f9 100644 --- a/website/www/site/content/en/blog/beam-2.22.0.md +++ b/website/www/site/content/en/blog/beam-2.22.0.md @@ -18,7 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. --> We are happy to present the new 2.22.0 release of Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2220-2020-06-08) for this release. +See the [download page](/get-started/downloads/#2220-2020-06-08) for this release. For more information on changes in 2.22.0, check out the [detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347144). diff --git a/website/www/site/content/en/blog/beam-2.23.0.md b/website/www/site/content/en/blog/beam-2.23.0.md index 49b095509866..7e825b53c21d 100644 --- a/website/www/site/content/en/blog/beam-2.23.0.md +++ b/website/www/site/content/en/blog/beam-2.23.0.md @@ -4,7 +4,7 @@ date: 2020-07-29 00:00:01 -0800 categories: - blog authors: - - Valentyn Tymofieiev + - tvalentyn --- We are happy to present the new 2.23.0 release of Apache Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2230-2020-07-29) for this release. +See the [download page](/get-started/downloads/#2230-2020-07-29) for this release. For more information on changes in 2.23.0, check out the [detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347145). diff --git a/website/www/site/content/en/blog/beam-2.24.0.md b/website/www/site/content/en/blog/beam-2.24.0.md index 69f000bc8766..3356f417b79b 100644 --- a/website/www/site/content/en/blog/beam-2.24.0.md +++ b/website/www/site/content/en/blog/beam-2.24.0.md @@ -4,7 +4,7 @@ date: 2020-09-18 00:00:01 -0800 categories: - blog authors: - - Daniel Oliveira + - danoliveira --- We are happy to present the new 2.24.0 release of Apache Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2240-2020-09-18) for this release. +See the [download page](/get-started/downloads/#2240-2020-09-18) for this release. For more information on changes in 2.24.0, check out the [detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347146). diff --git a/website/www/site/content/en/blog/beam-2.25.0.md b/website/www/site/content/en/blog/beam-2.25.0.md index 724eddf64570..e6eb3df07c16 100644 --- a/website/www/site/content/en/blog/beam-2.25.0.md +++ b/website/www/site/content/en/blog/beam-2.25.0.md @@ -4,7 +4,7 @@ date: 2020-10-23 14:00:00 -0800 categories: - blog authors: - - Robin Qiu + - robinyq --- We are happy to present the new 2.25.0 release of Apache Beam. This release includes both improvements and new functionality. -See the [download page](/get-started/downloads/#2250-2020-10-23) for this release. +See the [download page](/get-started/downloads/#2250-2020-10-23) for this release. For more information on changes in 2.25.0, check out the [detailed release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347147). diff --git a/website/www/site/content/en/blog/pattern-match-beam-sql.md b/website/www/site/content/en/blog/pattern-match-beam-sql.md index fb0620616ed0..409eedf0d782 100644 --- a/website/www/site/content/en/blog/pattern-match-beam-sql.md +++ b/website/www/site/content/en/blog/pattern-match-beam-sql.md @@ -6,7 +6,7 @@ aliases: categories: - blog authors: - - Mark-Zeng + - markzeng --- For more information on changes in {$RELEASE_VERSION}, check out the [detailed release notes]({$JIRA_RELEASE_NOTES}). @@ -1088,7 +1089,7 @@ please follow [the guide](https://help.github.com/articles/creating-a-personal-a All wheels should be published, in addition to the zip of the release source. (Signatures and hashes do _not_ need to be uploaded.) -### Deploy SDK docker images to DockerHub +### Deploy docker images to DockerHub * Script: [publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh) * Usage ``` @@ -1099,6 +1100,8 @@ Verify that: * Images with *latest* tag are pointing to current release by confirming 1. Digest of the image with *latest* tag is the same as the one with {RELEASE} tag. +(Optional) Clean up any unneeded local images afterward to save disk space. + ### Merge Website pull requests Merge all of the website pull requests diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index 2182b4b8f438..59fedfdeada5 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -281,6 +281,10 @@ to the key-values pairs specified in `columnsMapping`. Not all existing column families and qualifiers have to be provided to the schema. +Filters are only allowed by `key` field with single `LIKE` statement with +[RE2 Syntax](https://github.com/google/re2/wiki/Syntax) regex, e.g. +`SELECT * FROM table WHERE key LIKE '^key[012]{1}'` + ### Write Mode Supported for flat schema only. @@ -447,9 +451,14 @@ TBLPROPERTIES '{ server. * `topics`: Optional. Allows you to specify specific topics. * `format`: Optional. Allows you to specify the Kafka values format. Possible values are - {`csv`, `avro`, `json`, `proto`}. Defaults to `csv`. + {`csv`, `avro`, `json`, `proto`, `thrift`}. Defaults to `csv`. * `protoClass`: Optional. Use only when `format` is equal to `proto`. Allows you to specify full protocol buffer java class name. + * `thriftClass`: Optional. Use only when `format` is equal to `thrift`. Allows you to + specify full thrift java class name. + * `thriftProtocolFactoryClass`: Optional. Use only when `format` is equal to `thrift`. + Allows you to specify full class name of the `TProtocolFactory` to use for thrift + serialization. ### Read Mode @@ -472,6 +481,10 @@ Write Mode supports writing to a topic. * Beam attempts to parse JSON to match the schema. * Protocol buffers * Fields in the schema have to match the fields of the given `protoClass`. +* Thrift + * Fields in the schema have to match the fields of the given `thriftClass`. + * The `TProtocolFactory` used for thrift serialization must match the + provided `thriftProtocolFactoryClass`. ### Schema diff --git a/website/www/site/content/en/documentation/dsls/sql/overview.md b/website/www/site/content/en/documentation/dsls/sql/overview.md index 6c7aacd4cc91..0b0c03ad6c51 100644 --- a/website/www/site/content/en/documentation/dsls/sql/overview.md +++ b/website/www/site/content/en/documentation/dsls/sql/overview.md @@ -18,7 +18,7 @@ limitations under the License. # Beam SQL overview -Beam SQL allows a Beam user (currently only available in Beam Java) to query +Beam SQL allows a Beam user (currently only available in Beam Java and Python) to query bounded and unbounded `PCollections` with SQL statements. Your SQL query is translated to a `PTransform`, an encapsulated segment of a Beam pipeline. You can freely mix SQL `PTransforms` and other `PTransforms` in your pipeline. diff --git a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md index a4f61cc7b60d..e368826ea6d1 100644 --- a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md +++ b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md @@ -252,13 +252,18 @@ them into JSON `TableRow` objects. {{< paragraph class="language-py" >}} -To read from a BigQuery table using the Beam SDK for Python, apply a `Read` -transform on a `BigQuerySource`. Read returns a `PCollection` of dictionaries, +To read from a BigQuery table using the Beam SDK for Python, apply a `ReadFromBigQuery` +transfrom. `ReadFromBigQuery` returns a `PCollection` of dictionaries, where each element in the `PCollection` represents a single row in the table. Integer values in the `TableRow` objects are encoded as strings to match BigQuery's exported JSON format. {{< /paragraph >}} +{{< paragraph class="language-py" >}} +***Note:*** `BigQuerySource()` is deprecated as of Beam SDK 2.25.0. Before 2.25.0, to read from +a BigQuery table using the Beam SDK, you will apply a `Read` transform on a `BigQuerySource`. For example, +`beam.io.Read(beam.io.BigQuerySource(table_spec))`. +{{< /paragraph >}} ### Reading from a table @@ -293,7 +298,7 @@ the `fromQuery` method. {{< paragraph class="language-py" >}} If you don't want to read an entire table, you can supply a query string to -`BigQuerySource` by specifying the `query` parameter. +`ReadFromBigQuery` by specifying the `query` parameter. {{< /paragraph >}} {{< paragraph class="language-py" >}} @@ -327,10 +332,10 @@ such as column selection and predicate filter push-down which can allow more efficient pipeline execution. The Beam SDK for Java supports using the BigQuery Storage API when reading from -BigQuery. SDK versions before 2.24.0 support the BigQuery Storage API as an +BigQuery. SDK versions before 2.25.0 support the BigQuery Storage API as an [experimental feature](https://beam.apache.org/releases/javadoc/current/index.html?org/apache/beam/sdk/annotations/Experimental.html) and use the pre-GA BigQuery Storage API surface. Callers should migrate -pipelines which use the BigQuery Storage API to use SDK version 2.24.0 or later. +pipelines which use the BigQuery Storage API to use SDK version 2.25.0 or later. The Beam SDK for Python does not support the BigQuery Storage API. See [BEAM-10917](https://issues.apache.org/jira/browse/BEAM-10917)). @@ -360,7 +365,7 @@ GitHub](https://github.com/apache/beam/blob/master/examples/java/src/main/java/o The following code snippet reads with a query string. {{< highlight java >}} -// Snippet not yet available (BEAM-7034). +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java" bigquery_read_from_query_with_bigquery_storage_api >}} {{< /highlight >}} {{< highlight py >}} diff --git a/website/www/site/content/en/documentation/io/built-in/snowflake.md b/website/www/site/content/en/documentation/io/built-in/snowflake.md index cb674507eb10..590032edd752 100644 --- a/website/www/site/content/en/documentation/io/built-in/snowflake.md +++ b/website/www/site/content/en/documentation/io/built-in/snowflake.md @@ -404,7 +404,7 @@ data.apply( - `.toTable()` - Accepts the target Snowflake table name. - - Example: `.toTable("MY_TABLE)` + - Example: `.toTable("MY_TABLE")` - `.withStagingBucketName()` - Accepts a cloud bucket path ended with slash. @@ -452,7 +452,9 @@ AS COPY INTO stream_table from @streamstage; **Note**: -SnowflakeIO uses COPY statements behind the scenes to write (using [COPY to table](https://docs.snowflake.net/manuals/sql-reference/sql/copy-into-table.html)). StagingBucketName will be used to save CSV files which will end up in Snowflake. Those CSV files will be saved under the “stagingBucketName” path. +As mentioned before SnowflakeIO uses [SnowPipe REST calls](https://docs.snowflake.com/en/user-guide/data-load-snowpipe.html) +behind the scenes for writing from unbounded sources. StagingBucketName will be used to save CSV files which will end up in Snowflake. +SnowflakeIO is not going to delete created CSV files from path under the “stagingBucketName” either during or after finishing streaming. **Optional** for streaming: - `.withFlushTimeLimit()` diff --git a/website/www/site/content/en/documentation/programming-guide.md b/website/www/site/content/en/documentation/programming-guide.md index 477152c23cb1..33d92df69473 100644 --- a/website/www/site/content/en/documentation/programming-guide.md +++ b/website/www/site/content/en/documentation/programming-guide.md @@ -2511,7 +2511,7 @@ public class TransactionPojo { public final double purchaseAmount; @SchemaCreate public TransactionPojo(String bank, double purchaseAmount) { - this.bank = bank. + this.bank = bank; this.purchaseAmount = purchaseAmount; } } @@ -3085,7 +3085,7 @@ The resulting schema is the following:

    -Each resulting row contains one Review and one Review that matched the join condition. +Each resulting row contains one Transaction and one Review that matched the join condition. If the fields to match in the two schemas have different names, then the on function can be used. For example, if the Review schema named those fields differently than the Transaction schema, then we could write the following: @@ -4519,6 +4519,8 @@ registered, then Beam will automatically infer the coder for the state value. Ot specified when creating the ValueState. For example, the following ParDo creates a single state variable that accumulates the number of elements seen. +Note: `ValueState` is called `ReadModifyWriteState` in the Python SDK. + {{< highlight java >}} PCollection> perUser = readPerUser(); perUser.apply(ParDo.of(new DoFn, OutputT>() { @@ -4544,6 +4546,20 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} + +{{< highlight py >}} +class ReadModifyWriteStateDoFn(DoFn): + STATE_SPEC = ReadModifyWriteStateSpec('num_elements', VarIntCoder()) + + def process(self, element, state=DoFn.StateParam(STATE_SPEC)): + # Read the number element seen so far for this user key. + current_value = state.read() or 0 + state.write(current_value+1) + +_ = (p | 'Read per user' >> ReadPerUser() + | 'state pardo' >> beam.ParDo(ReadModifyWriteStateDoFn())) +{{< /highlight >}} + #### CombiningState `CombiningState` allows you to create a state object that is updated using a Beam combiner. For example, the previous @@ -4561,11 +4577,11 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class CombiningStateDoFn(DoFn): SUM_TOTAL = CombiningValueStateSpec('total', sum) - def process(self, element, state=SoFn.StateParam(SUM_TOTAL)): + def process(self, element, state=DoFn.StateParam(SUM_TOTAL)): state.add(1) _ = (p | 'Read per user' >> ReadPerUser() @@ -4599,7 +4615,7 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class BagStateDoFn(DoFn): ALL_ELEMENTS = BagStateSpec('buffer', coders.VarIntCoder()) @@ -4639,6 +4655,11 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} + +{{< highlight py >}} +This is not supported yet, see BEAM-11506. +{{< /highlight >}} + If however there are code paths in which the states are not fetched, then annotating with @AlwaysFetched will add unnecessary fetching for those paths. In this case, the readLater method allows the runner to know that the state will be read in the future, allowing multiple state reads to be batched together. @@ -4703,7 +4724,7 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class EventTimerDoFn(DoFn): ALL_ELEMENTS = BagStateSpec('buffer', coders.VarIntCoder()) TIMER = TimerSpec('timer', TimeDomain.WATERMARK) @@ -4751,7 +4772,7 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class ProcessingTimerDoFn(DoFn): ALL_ELEMENTS = BagStateSpec('buffer', coders.VarIntCoder()) TIMER = TimerSpec('timer', TimeDomain.REAL_TIME) @@ -4775,12 +4796,15 @@ _ = (p | 'Read per user' >> ReadPerUser() #### 11.3.3. Dynamic timer tags {#dynamic-timer-tags} -Beam also supports dynamically setting a timer tag using `TimerMap`. This allows for setting multiple different timers +Beam also supports dynamically setting a timer tag using `TimerMap` in the Java SDK. This allows for setting multiple different timers in a `DoFn` and allowing for the timer tags to be dynamically chosen - e.g. based on data in the input elements. A timer with a specific tag can only be set to a single timestamp, so setting the timer again has the effect of overwriting the previous expiration time for the timer with that tag. Each `TimerMap` is identified with a timer family id, and timers in different timer families are independent. +In the Python SDK, a dynamic timer tag can be specified while calling `set()` or `clear()`. By default, the timer +tag is an empty string if not specified. + {{< highlight java >}} PCollection> perUser = readPerUser(); perUser.apply(ParDo.of(new DoFn, OutputT>() { @@ -4800,9 +4824,33 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} -{{< highlight python >}} -To be supported, See BEAM-9602 +{{< highlight py >}} +class TimerDoFn(DoFn): + ALL_ELEMENTS = BagStateSpec('buffer', coders.VarIntCoder()) + TIMER = TimerSpec('timer', TimeDomain.REAL_TIME) + + def process(self, + element_pair, + buffer = DoFn.StateParam(ALL_ELEMENTS), + timer = DoFn.TimerParam(TIMER)): + buffer.add(element_pair[1]) + # Set a timer to go off 30 seconds in the future with dynamic timer tag 'first_timer'. + # And set a timer to go off 60 seconds in the future with dynamic timer tag 'second_timer'. + timer.set(Timestamp.now() + Duration(seconds=30), dynamic_timer_tag='first_timer') + timer.set(Timestamp.now() + Duration(seconds=60), dynamic_timer_tag='second_timer') + # Note that a timer can also be explicitly cleared if previously set with a dynamic timer tag: + # timer.clear(dynamic_timer_tag=...) + + @on_timer(TIMER) + def expiry_callback(self, buffer = DoFn.StateParam(ALL_ELEMENTS), timer_tag=DoFn.DynamicTimerTagParam): + # Process timer, the dynamic timer tag associated with expiring timer can be read back with DoFn.DynamicTimerTagParam. + buffer.clear() + yield (timer_tag, 'fired') + +_ = (p | 'Read per user' >> ReadPerUser() + | 'ProcessingTime timer pardo' >> beam.ParDo(TimerDoFn())) {{< /highlight >}} + #### 11.3.4. Timer output timestamps {#timer-output-timestamps} By default, event-time timers will hold the output watermark of the `ParDo` to the timestamp of the timer. This means @@ -4906,6 +4954,10 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} +{{< highlight py >}} +Timer output timestamps is not yet supported in Python SDK. See BEAM-11507. +{{< /highlight >}} + ### 11.4. Garbage collecting state {#garbage-collecting-state} Per-key state needs to be garbage collected, or eventually the increasing size of state may negatively impact performance. There are two common strategies for garbage collecting state. @@ -4934,7 +4986,7 @@ perUser.apply(Window.into(CalendarWindows.days(1) })); {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class StateDoFn(DoFn): ALL_ELEMENTS = BagStateSpec('buffer', coders.VarIntCoder()) @@ -4995,7 +5047,7 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { } {{< /highlight >}} -{{< highlight python >}} +{{< highlight py >}} class UserDoFn(DoFn): ALL_ELEMENTS = BagStateSpec('state', coders.VarIntCoder()) MAX_TIMESTAMP = CombiningValueStateSpec('max_timestamp_seen', max) @@ -5052,7 +5104,7 @@ PCollection> eventsPerLinkId = readEvents() .apply(WithKeys.of(Event::getLinkId).withKeyType(TypeDescriptors.strings())); -perUser.apply(ParDo.of(new DoFn, JoinedEvent>() { +eventsPerLinkId.apply(ParDo.of(new DoFn, JoinedEvent>() { // Store the view event. @StateId("view") private final StateSpec> viewState = StateSpecs.value(); // Store the click event. @@ -5114,6 +5166,57 @@ perUser.apply(ParDo.of(new DoFn, JoinedEvent>() { })); {{< /highlight >}} +{{< highlight py >}} +class JoinDoFn(DoFn): + # stores the view event. + VIEW_STATE_SPEC = ReadModifyWriteStateSpec('view', EventCoder()) + # stores the click event. + CLICK_STATE_SPEC = ReadModifyWriteStateSpec('click', EventCoder()) + # The maximum element timestamp value seen so far. + MAX_TIMESTAMP = CombiningValueStateSpec('max_timestamp_seen', max) + # Timer that fires when an hour goes by with an incomplete join. + GC_TIMER = TimerSpec('gc', TimeDomain.WATERMARK) + + def process(self, + element, + view=DoFn.StateParam(VIEW_STATE_SPEC), + click=DoFn.StateParam(CLICK_STATE_SPEC), + max_timestamp_seen=DoFn.StateParam(MAX_TIMESTAMP), + ts=DoFn.TimestampParam, + gc=DoFn.TimerParam(GC_TIMER)): + event = element + if event.type == 'view': + view.write(event) + else: + click.write(event) + + previous_view = view.read() + previous_click = click.read() + + # We've seen both a view and a click. Output a joined event and clear state. + if previous_view and previous_click: + yield (previous_view, previous_click) + view.clear() + click.clear() + max_timestamp_seen.clear() + else: + max_timestamp_seen.add(ts) + gc.set(max_timestamp_seen.read() + Duration(seconds=3600)) + + @on_timer(GC_TIMER) + def gc_callback(self, + view=DoFn.StateParam(VIEW_STATE_SPEC), + click=DoFn.StateParam(CLICK_STATE_SPEC), + max_timestamp_seen=DoFn.StateParam(MAX_TIMESTAMP)): + view.clear() + click.clear() + max_timestamp_seen.clear() + + +_ = (p | 'EventsPerLinkId' >> ReadPerLinkEvents() + | 'Join DoFn' >> beam.ParDo(JoinDoFn())) +{{< /highlight >}} + #### 11.5.2. Batching RPCs {#batching-rpcs} In this example, input elements are being forwarded to an external RPC service. The RPC accepts batch requests - @@ -5155,6 +5258,31 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} +{{< highlight py >}} +class BufferDoFn(DoFn): + BUFFER = BagStateSpec('buffer', EventCoder()) + IS_TIMER_SET = ReadModifyWriteStateSpec('is_timer_set', BooleanCoder()) + OUTPUT = TimerSpec('output', TimeDomain.REAL_TIME) + + def process(self, + buffer=DoFn.StateParam(BUFFER), + is_timer_set=DoFn.StateParam(IS_TIMER_SET), + timer=DoFn.TimerParam(OUTPUT)): + buffer.add(element) + if not is_timer_set.read(): + timer.set(Timestamp.now() + Duration(seconds=10)) + is_timer_set.write(True) + + @on_timer(OUTPUT) + def output_callback(self, + buffer=DoFn.StateParam(BUFFER), + is_timer_set=DoFn.StateParam(IS_TIMER_SET)): + send_rpc(list(buffer.read())) + buffer.clear() + is_timer_set.clear() + +{{< /highlight >}} + ## 12. Splittable `DoFns` {#splittable-dofns} A Splittable `DoFn` (SDF) enables users to create modular components containing I/Os (and some advanced diff --git a/website/www/site/content/en/documentation/runners/direct.md b/website/www/site/content/en/documentation/runners/direct.md index 0168dccdd30e..1249aa9a2860 100644 --- a/website/www/site/content/en/documentation/runners/direct.md +++ b/website/www/site/content/en/documentation/runners/direct.md @@ -57,6 +57,8 @@ Here are some resources with information about how to test your pipelines. ## Pipeline options for the Direct Runner +For general instructions on how to set pipeline options, see the [programming guide](/documentation/programming-guide/#configuring-pipeline-options). + When executing your pipeline from the command-line, set `runner` to `direct` or `DirectRunner`. The default values for the other pipeline options are generally sufficient. See the reference documentation for the @@ -74,105 +76,41 @@ Local execution is limited by the memory available in your local environment. It If your pipeline uses an unbounded data source or sink, you must set the `streaming` option to `true`. -{:.language-py} -### Execution Mode +### Parallel execution -{:.language-py} +{{< paragraph class="language-py" >}} Python [FnApiRunner](https://beam.apache.org/contribute/runner-guide/#the-fn-api) supports multi-threading and multi-processing mode. +{{< /paragraph >}} + +#### Setting parallelism -{:.language-py} -Setting parallelism +{{< paragraph class="language-java" >}} +The number of worker threads is defined by the `targetParallelism` pipeline option. +By default, `targetParallelism` is the greater of the number of available processors and 3. +{{< /paragraph >}} -{:.language-py} -Number of threads or subprocesses is defined by setting the `direct_num_workers` option. +{{< paragraph class="language-py" >}} +Number of threads or subprocesses is defined by setting the `direct_num_workers` pipeline option. From 2.22.0, `direct_num_workers = 0` is supported. When `direct_num_workers` is set to 0, it will set the number of threads/subprocess to the number of cores of the machine where the pipeline is running. +{{< /paragraph >}} -{:.language-py} -* There are several ways to set this option. -```py -python wordcount.py --input xx --output xx --direct_num_workers 2 -``` - -{:.language-py} -* Setting with `PipelineOptions`. -```py -from apache_beam.options.pipeline_options import PipelineOptions -pipeline_options = PipelineOptions(['--direct_num_workers', '2']) -``` - -{:.language-py} -* Adding to existing `PipelineOptions`. -```py -from apache_beam.options.pipeline_options import DirectOptions -pipeline_options = PipelineOptions(xxx) -pipeline_options.view_as(DirectOptions).direct_num_workers = 2 -``` - -{:.language-py} +{{< paragraph class="language-py" >}} Setting running mode +{{< /paragraph >}} -{:.language-py} -From 2.19, a new option was added to set running mode. We can use `direct_running_mode` option to set the running mode. +{{< paragraph class="language-py" >}} +In Beam 2.19.0 and newer, you can use the `direct_running_mode` pipeline option to set the running mode. `direct_running_mode` can be one of [`'in_memory'`, `'multi_threading'`, `'multi_processing'`]. +{{< /paragraph >}} -{:.language-py} +{{< paragraph class="language-py" >}} in_memory: Runner and workers' communication happens in memory (not through gRPC). This is a default mode. +{{< /paragraph >}} -{:.language-py} +{{< paragraph class="language-py" >}} multi_threading: Runner and workers communicate through gRPC and each worker runs in a thread. +{{< /paragraph >}} -{:.language-py} +{{< paragraph class="language-py" >}} multi_processing: Runner and workers communicate through gRPC and each worker runs in a subprocess. - -{:.language-py} -Same as other options, `direct_running_mode` can be passed through CLI or set with `PipelineOptions`. - -{:.language-py} -For the versions before 2.19.0, the running mode should be set with `FnApiRunner()`. Please refer following examples. - -{:.language-py} -#### Running with multi-threading mode -```py -import argparse - -import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.runners.portability import fn_api_runner -from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.portability import python_urns - -parser = argparse.ArgumentParser() -parser.add_argument(...) -known_args, pipeline_args = parser.parse_known_args(argv) -pipeline_options = PipelineOptions(pipeline_args) - -p = beam.Pipeline(options=pipeline_options, - runner=fn_api_runner.FnApiRunner( - default_environment=beam_runner_api_pb2.Environment( - urn=python_urns.EMBEDDED_PYTHON_GRPC))) -``` - -{:.language-py} -#### Running with multi-processing mode -```py -import argparse -import sys - -import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.runners.portability import fn_api_runner -from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.portability import python_urns - -parser = argparse.ArgumentParser() -parser.add_argument(...) -known_args, pipeline_args = parser.parse_known_args(argv) -pipeline_options = PipelineOptions(pipeline_args) - -p = beam.Pipeline(options=pipeline_options, - runner=fn_api_runner.FnApiRunner( - default_environment=beam_runner_api_pb2.Environment( - urn=python_urns.SUBPROCESS_SDK, - payload=b'%s -m apache_beam.runners.worker.sdk_worker_main' - % sys.executable.encode('ascii')))) -``` +{{< /paragraph >}} diff --git a/website/www/site/content/en/documentation/runners/flink.md b/website/www/site/content/en/documentation/runners/flink.md index d173f70a003a..e1f814c19177 100644 --- a/website/www/site/content/en/documentation/runners/flink.md +++ b/website/www/site/content/en/documentation/runners/flink.md @@ -326,7 +326,45 @@ To find out which version of Flink is compatible with Beam please see the table Artifact Id - ≥ 2.21.0 + ≥ 2.27.0 + 1.12.x * + beam-runners-flink-1.12 + + + 1.11.x * + beam-runners-flink-1.11 + + + 1.10.x + beam-runners-flink-1.10 + + + 1.9.x + beam-runners-flink-1.9 + + + 1.8.x + beam-runners-flink-1.8 + + + 2.25.0 - 2.26.0 + 1.11.x * + beam-runners-flink-1.11 + + + 1.10.x + beam-runners-flink-1.10 + + + 1.9.x + beam-runners-flink-1.9 + + + 1.8.x + beam-runners-flink-1.8 + + + 2.21.0 - 2.24.0 1.10.x beam-runners-flink-1.10 @@ -421,6 +459,8 @@ To find out which version of Flink is compatible with Beam please see the table +* This version does not have a published docker image for the Flink Job Service. + For retrieving the right Flink version, see the [Flink downloads page](https://flink.apache.org/downloads.html). For more information, the [Flink Documentation](https://ci.apache.org/projects/flink/flink-docs-stable/) can be helpful. diff --git a/website/www/site/content/en/documentation/runners/samza.md b/website/www/site/content/en/documentation/runners/samza.md index 355d8c0dfb27..fc9325485206 100644 --- a/website/www/site/content/en/documentation/runners/samza.md +++ b/website/www/site/content/en/documentation/runners/samza.md @@ -161,6 +161,11 @@ When executing your pipeline with the Samza Runner, you can use the following pi The maximum number of messages to buffer for a given system. 5000 + + eventTimerBufferSize + The maximum number of event-time timers to buffer in memory for a PTransform + 5000 + maxSourceParallelism The maximum parallelism allowed for any data source. diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md index ea7cc17d4bfe..952a04c7a9d8 100644 --- a/website/www/site/content/en/documentation/runtime/environments.md +++ b/website/www/site/content/en/documentation/runtime/environments.md @@ -17,147 +17,252 @@ limitations under the License. # Container environments -The Beam SDK runtime environment is isolated from other runtime systems because the SDK runtime environment is [containerized](https://s.apache.org/beam-fn-api-container-contract) with [Docker](https://www.docker.com/). This means that any execution engine can run the Beam SDK. +The Beam SDK runtime environment can be [containerized](https://www.docker.com/resources/what-container) with [Docker](https://www.docker.com/) to isolate it from other runtime systems. To learn more about the container environment, read the Beam [SDK Harness container contract](https://s.apache.org/beam-fn-api-container-contract). -This page describes how to customize, build, and push Beam SDK container images. +Prebuilt SDK container images are released per supported language during Beam releases and pushed to [Docker Hub](https://hub.docker.com/search?q=apache%2Fbeam&type=image). -Before you begin, install [Docker](https://www.docker.com/) on your workstation. +## Custom containers -## Customizing container images +You may want to customize container images for many reasons, including: -You can add extra dependencies to container images so that you don't have to supply the dependencies to execution engines. +* Pre-installing additional dependencies +* Launching third-party software in the worker environment +* Further customizing the execution environment -To customize a container image, either: -* [Write a new](#writing-new-dockerfiles) [Dockerfile](https://docs.docker.com/engine/reference/builder/) on top of the original. -* [Modify](#modifying-dockerfiles) the [original Dockerfile](https://github.com/apache/beam/blob/master/sdks/python/container/Dockerfile) and reimage the container. + This guide describes how to create and use customized containers for the Beam SDK. -It's often easier to write a new Dockerfile. However, by modifying the original Dockerfile, you can customize anything (including the base OS). +### Prerequisites -### Writing new Dockerfiles on top of the original {#writing-new-dockerfiles} +* This guide requires building images using Docker. [Install Docker locally](https://docs.docker.com/get-docker/). Some CI/CD platforms like [Google Cloud Build](https://cloud.google.com/cloud-build/docs/building/build-containers) also provide the ability to build images using Docker. +* For remote execution engines/runners, have a container registry to host your custom container image. Options include [Docker Hub](https://hub.docker.com/) or a "self-hosted" repository, including cloud-specific container registries like [Google Container Registry](https://cloud.google.com/container-registry) (GCR) or [Amazon Elastic Container Registry](https://aws.amazon.com/ecr/) (ECR). Make sure your registry can be accessed by your execution engine or runner. + +> **NOTE**: On Nov 20, 2020, Docker Hub put [rate limits](https://www.docker.com/increase-rate-limits) into effect for anonymous and free authenticated use, which may impact larger pipelines that pull containers several times. + +For optimal user experience, we also recommend you use the latest released version of Beam. + +### Building and pushing custom containers + +Beam [SDK container images](https://hub.docker.com/search?q=apache%2Fbeam&type=image) are built from Dockerfiles checked into the [Github](https://github.com/apache/beam) repository and published to Docker Hub for every release. You can build customized containers in one of two ways: + +1. **[Writing a new](#writing-new-dockerfiles) Dockerfile based on a released container image**. This is sufficient for simple additions to the image, such as adding artifacts or environment variables. +2. **[Modifying](#modifying-dockerfiles) a source Dockerfile in [Beam](https://github.com/apache/beam)**. This method requires building from Beam source but allows for greater customization of the container (including replacement of artifacts or base OS/language versions). + +#### Writing a new Dockerfile based on an existing published container image {#writing-new-dockerfiles} + +1. Create a new Dockerfile that designates a base image using the [FROM instruction](https://docs.docker.com/engine/reference/builder/#from). -1. Pull a [prebuilt SDK container image](https://hub.docker.com/search?q=apache%2Fbeam&type=image) for your [target](https://docs.docker.com/docker-hub/repos/#searching-for-repositories) language and version. The following example pulls the latest Python SDK: ``` -docker pull apache/beam_python3.7_sdk +FROM apache/beam_python3.7_sdk:2.25.0 + +ENV FOO=bar +COPY /src/path/to/file /dest/path/to/file/ ``` -2. [Write a new Dockerfile](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/) that [designates](https://docs.docker.com/engine/reference/builder/#from) the original as its [parent](https://docs.docker.com/glossary/?term=parent%20image). -3. [Build](#building-container-images) a child image. -### Modifying the original Dockerfile {#modifying-dockerfiles} +This `Dockerfile` uses the prebuilt Python 3.7 SDK container image [`beam_python3.7_sdk`](https://hub.docker.com/r/apache/beam_python3.7_sdk) tagged at (SDK version) `2.25.0`, and adds an additional environment variable and file to the image. + + +2. [Build](https://docs.docker.com/engine/reference/commandline/build/) and [push](https://docs.docker.com/engine/reference/commandline/push/) the image using Docker. + + ``` + export BASE_IMAGE="apache/beam_python3.7_sdk:2.25.0" + export IMAGE_NAME="myremoterepo/mybeamsdk" + export TAG="latest" + + # Optional - pull the base image into your local Docker daemon to ensure + # you have the most up-to-date version of the base image locally. + docker pull "${BASE_IMAGE}" + + docker build -f Dockerfile -t "${IMAGE_NAME}:${TAG}" . + ``` + +3. If your runner is running remotely, retag and [push](https://docs.docker.com/engine/reference/commandline/push/) the image to the appropriate repository. + + ``` + docker push "${IMAGE_NAME}:${TAG}" + ``` + +4. After pushing a container image, verify the remote image ID and digest matches the local image ID and digest, output from `docker build` or `docker images`. + +#### Modifying a source Dockerfile in Beam {#modifying-dockerfiles} + +This method requires building image artifacts from Beam source. For additional instructions on setting up your development environment, see the [Contribution guide](/contribute/#development-setup). + +>**NOTE**: It is recommended that you start from a stable release branch (`release-X.XX.X`) corresponding to the same version of the SDK to run your pipeline. Differences in SDK version may result in unexpected errors. + +1. Clone the `beam` repository. + + ``` + export BEAM_SDK_VERSION="2.26.0" + git clone https://github.com/apache/beam.git + cd beam + + # Save current directory as working directory + export BEAM_WORKDIR=$PWD + + git checkout origin/release-$BEAM_SDK_VERSION + ``` + +2. Customize the `Dockerfile` for a given language, typically `sdks//container/Dockerfile` directory (e.g. the [Dockerfile for Python](https://github.com/apache/beam/blob/master/sdks/python/container/Dockerfile). If you're adding dependencies from [PyPI](https://pypi.org/), use [`base_image_requirements.txt`](https://github.com/apache/beam/blob/master/sdks/python/container/base_image_requirements.txt) instead. + +3. Return to the root Beam directory and run the Gradle `docker` target for your image. + + ``` + cd $BEAM_WORKDIR + + # The default repository of each SDK + ./gradlew :sdks:java:container:java8:docker + ./gradlew :sdks:java:container:java11:docker + ./gradlew :sdks:go:container:docker + ./gradlew :sdks:python:container:py36:docker + ./gradlew :sdks:python:container:py37:docker + ./gradlew :sdks:python:container:py38:docker + + # Shortcut for building all Python SDKs + ./gradlew :sdks:python:container buildAll + ``` + +4. Verify the images you built were created by running `docker images`. + + ``` + $> docker images --digests + REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE + apache/beam_java8_sdk latest sha256:... ... 1 min ago ... + apache/beam_java11_sdk latest sha256:... ... 1 min ago ... + apache/beam_python3.6_sdk latest sha256:... ... 1 min ago ... + apache/beam_python3.7_sdk latest sha256:... ... 1 min ago ... + apache/beam_python3.8_sdk latest sha256:... ... 1 min ago ... + apache/beam_go_sdk latest sha256:... ... 1 min ago ... + ``` + +5. If your runner is running remotely, retag the image and [push](https://docs.docker.com/engine/reference/commandline/push/) the image to your repository. You can skip this step if you provide a custom repo/tag as [additional parameters](#additional-build-parameters). + + ``` + export BEAM_SDK_VERSION="2.26.0" + export IMAGE_NAME="gcr.io/my-gcp-project/beam_python3.7_sdk" + export TAG="${BEAM_SDK_VERSION}-custom" + + docker tag apache/beam_python3.7_sdk "${IMAGE_NAME}:${TAG}" + docker push "${IMAGE_NAME}:${TAG}" + ``` + +6. After pushing a container image, verify the remote image ID and digest matches the local image ID and digest output from `docker_images --digests`. + +#### Additional build parameters{#additional-build-parameters} + +The docker Gradle task defines a default image repository and [tag](https://docs.docker.com/engine/reference/commandline/tag/) is the SDK version defined at [gradle.properties](https://github.com/apache/beam/blob/master/gradle.properties). The default repository is the Docker Hub `apache` namespace, and the default tag is the [SDK version](https://github.com/apache/beam/blob/master/gradle.properties) defined at gradle.properties. + +You can specify a different repository or tag for built images by providing parameters to the build task. For example: -1. Clone the `beam` repository: ``` -git clone https://github.com/apache/beam.git +./gradlew :sdks:python:container:py36:docker -Pdocker-repository-root="example-repo" -Pdocker-tag="2.26.0-custom" ``` -2. Customize the [Dockerfile](https://github.com/apache/beam/blob/master/sdks/python/container/Dockerfile). If you're adding dependencies from [PyPI](https://pypi.org/), use [`base_image_requirements.txt`](https://github.com/apache/beam/blob/master/sdks/python/container/base_image_requirements.txt) instead. -3. [Reimage](#building-container-images) the container. -### Testing customized images +builds the Python 3.6 container and tags it as `example-repo/beam_python3.6_sdk:2.26.0-custom`. -To test a customized image locally, run a pipeline with PortableRunner and set the `--environment_config` flag to the image path: +From Beam 2.21.0 and later, a `docker-pull-licenses` flag was introduced to add licenses/notices for third party dependencies to the docker images. For example: + +``` +./gradlew :sdks:java:container:java8:docker -Pdocker-pull-licenses +``` +creates a Java 8 SDK image with appropriate licenses in `/opt/apache/beam/third_party_licenses/`. + +By default, no licenses/notices are added to the docker images. + + +## Running pipelines with custom container images {#running-pipelines} + +The common method for providing a container image requires using the +PortableRunner flag `--environment_config` as supported by the Portable +Runner or by runners supported PortableRunner flags. +Other runners, such as Dataflow, support specifying containers with different flags. {{< highlight class="runner-direct" >}} +export IMAGE="my-repo/beam_python_sdk_custom" +export TAG="X.Y.Z" +export IMAGE_URL = "${IMAGE}:${TAG}" + python -m apache_beam.examples.wordcount \ --input=/path/to/inputfile \ --output /path/to/write/counts \ --runner=PortableRunner \ --job_endpoint=embed \ ---environment_config=path/to/container/image +--environment_type="DOCKER" \ +--environment_config="${IMAGE_URL}" {{< /highlight >}} {{< highlight class="runner-flink-local" >}} -# Start a Flink job server on localhost:8099 -./gradlew :runners:flink:1.8:job-server:runShadow +export IMAGE="my-repo/beam_python_sdk_custom" +export TAG="X.Y.Z" +export IMAGE_URL = "${IMAGE}:${TAG}" -# Run a pipeline on the Flink job server +# Run a pipeline using the FlinkRunner which starts a Flink job server. python -m apache_beam.examples.wordcount \ --input=/path/to/inputfile \ ---output=/path/to/write/counts \ ---runner=PortableRunner \ ---job_endpoint=localhost:8099 \ ---environment_config=path/to/container/image +--output=path/to/write/counts \ +--runner=FlinkRunner \ +--environment_type="DOCKER" \ +--environment_config="${IMAGE_URL}" {{< /highlight >}} {{< highlight class="runner-spark-local" >}} -# Start a Spark job server on localhost:8099 -./gradlew :runners:spark:job-server:runShadow +export IMAGE="my-repo/beam_python_sdk_custom" +export TAG="X.Y.Z" +export IMAGE_URL = "${IMAGE}:${TAG}" -# Run a pipeline on the Spark job server +# Run a pipeline using the SparkRunner which starts the Spark job server python -m apache_beam.examples.wordcount \ --input=/path/to/inputfile \ --output=path/to/write/counts \ ---runner=PortableRunner \ ---job_endpoint=localhost:8099 \ ---environment_config=path/to/container/image +--runner=SparkRunner \ +--environment_type="DOCKER" \ +--environment_config="${IMAGE_URL}" {{< /highlight >}} -## Building container images - -To build Beam SDK container images: - -1. Navigate to the root directory of the local copy of your Apache Beam. -2. Run Gradle with the `docker` target. If you're [building a child image](#writing-new-dockerfiles), set the optional `--file` flag to the new Dockerfile. If you're [building an image from an original Dockerfile](#modifying-dockerfiles), ignore the `--file` flag: - -``` -# The default repository of each SDK -./gradlew [--file=path/to/new/Dockerfile] :sdks:java:container:java8:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:java:container:java11:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:go:container:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:python:container:py2:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:python:container:py35:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:python:container:py36:docker -./gradlew [--file=path/to/new/Dockerfile] :sdks:python:container:py37:docker - -# Shortcut for building all four Python SDKs -./gradlew [--file=path/to/new/Dockerfile] :sdks:python:container buildAll -``` - -From 2.21.0, `docker-pull-licenses` tag was introduced. Licenses/notices of third party dependencies will be added to the docker images when `docker-pull-licenses` was set. -For example, `./gradlew :sdks:java:container:java8:docker -Pdocker-pull-licenses`. The files are added to `/opt/apache/beam/third_party_licenses/`. -By default, no licenses/notices are added to the docker images. - -To examine the containers that you built, run `docker images` from anywhere in the command line. If you successfully built all of the container images, the command prints a table like the following: -``` -REPOSITORY TAG IMAGE ID CREATED SIZE -apache/beam_java8_sdk latest ... 2 weeks ago ... -apache/beam_java11_sdk latest ... 2 weeks ago ... -apache/beam_python2.7_sdk latest ... 2 weeks ago ... -apache/beam_python3.5_sdk latest ... 2 weeks ago ... -apache/beam_python3.6_sdk latest ... 2 weeks ago ... -apache/beam_python3.7_sdk latest ... 2 weeks ago ... -apache/beam_go_sdk latest ... 2 weeks ago ... -``` - -### Overriding default Docker targets - -The default [tag](https://docs.docker.com/engine/reference/commandline/tag/) is sdk_version defined at [gradle.properties](https://github.com/apache/beam/blob/master/gradle.properties) and the default repositories are in the Docker Hub `apache` namespace. -The `docker` command-line tool implicitly [pushes container images](#pushing-container-images) to this location. - -To tag a local image, set the `docker-tag` option when building the container. The following command tags a Python SDK image with a date. -``` -./gradlew :sdks:python:container:py36:docker -Pdocker-tag=2019-10-04 -``` +{{< highlight class="runner-dataflow" >}} +export GCS_PATH="gs://my-gcs-bucket" +export GCP_PROJECT="my-gcp-project" +export REGION="us-central1" -To change the repository, set the `docker-repository-root` option to a new location. The following command sets the `docker-repository-root` -to a repository named `example-repo` on Docker Hub. -``` -./gradlew :sdks:python:container:py36:docker -Pdocker-repository-root=example-repo -``` - -## Pushing container images +# By default, the Dataflow runner has access to the GCR images +# under the same project. +export IMAGE="my-repo/beam_python_sdk_custom" +export TAG="X.Y.Z" +export IMAGE_URL = "${IMAGE}:${TAG}" -After [building a container image](#building-container-images), you can store it in a remote Docker repository. +# Run a pipeline on Dataflow. +# This is a Python batch pipeline, so to run on Dataflow Runner V2 +# you must specify the experiment "use_runner_v2" -The following steps push a Python3.6 SDK image to the [`docker-root-repository` value](#overriding-default-docker-targets). -Please log in to the destination repository as needed. +python -m apache_beam.examples.wordcount \ + --input gs://dataflow-samples/shakespeare/kinglear.txt \ + --output "${GCS_PATH}/counts" \ + --runner DataflowRunner \ + --project $GCP_PROJECT \ + --region $REGION \ + --temp_location "${GCS_PATH}/tmp/" \ + --experiment=use_runner_v2 \ + --worker_harness_container_image=$IMAGE_URL -Upload it to the remote repository: -``` -docker push example-repo/beam_python3.6_sdk -``` +{{< /highlight >}} -To download the image again, run `docker pull`: -``` -docker pull example-repo/beam_python3.6_sdk -``` -> **Note**: After pushing a container image, the remote image ID and digest match the local image ID and digest. +### Troubleshooting + +The following section describes some common issues to consider +when you encounter unexpected errors running Beam pipelines with +custom containers. + +* Differences in language and SDK version between the container SDK and + pipeline SDK may result in unexpected errors due to incompatibility. For best + results, make sure to use the same stable SDK version for your base container + and when running your pipeline. +* If you are running into unexpected errors when using remote containers, + make sure that your container exists in the remote repository and can be + accessed by any third-party service, if needed. +* Local runners attempt to pull remote images and default to local + images. If an image cannot be pulled locally (by the docker daemon), + you may see an log message like: + ``` + Error response from daemon: manifest for remote.repo/beam_python3.7_sdk:2.25.0-custom not found: manifest unknown: ... + INFO:apache_beam.runners.portability.fn_api_runner.worker_handlers:Unable to pull image... + ``` diff --git a/website/www/site/content/en/get-started/beam-overview.md b/website/www/site/content/en/get-started/beam-overview.md index baf8610463fc..40aacb01bc37 100644 --- a/website/www/site/content/en/get-started/beam-overview.md +++ b/website/www/site/content/en/get-started/beam-overview.md @@ -23,7 +23,7 @@ limitations under the License. Apache Beam is an open source, unified model for defining both batch and streaming data-parallel processing pipelines. Using one of the open source Beam SDKs, you build a program that defines the pipeline. The pipeline is then executed by one of Beam's supported **distributed processing back-ends**, which include [Apache Flink](https://flink.apache.org), [Apache Spark](http://spark.apache.org), and [Google Cloud Dataflow](https://cloud.google.com/dataflow). -Beam is particularly useful for [Embarrassingly Parallel](https://en.wikipedia.org/wiki/Embarassingly_parallel) data processing tasks, in which the problem can be decomposed into many smaller bundles of data that can be processed independently and in parallel. You can also use Beam for Extract, Transform, and Load (ETL) tasks and pure data integration. These tasks are useful for moving data between different storage media and data sources, transforming data into a more desirable format, or loading data onto a new system. +Beam is particularly useful for [embarrassingly parallel](https://en.wikipedia.org/wiki/Embarassingly_parallel) data processing tasks, in which the problem can be decomposed into many smaller bundles of data that can be processed independently and in parallel. You can also use Beam for Extract, Transform, and Load (ETL) tasks and pure data integration. These tasks are useful for moving data between different storage media and data sources, transforming data into a more desirable format, or loading data onto a new system. ## Apache Beam SDKs @@ -31,9 +31,9 @@ The Beam SDKs provide a unified programming model that can represent and transfo Beam currently supports the following language-specific SDKs: -- Java ![Java logo](/images/logos/sdks/java.png) -- Python ![Python logo](/images/logos/sdks/python.png) -- Go Go logo +- [Apache Beam Java SDK](/documentation/sdks/java) ![Java logo](/images/logos/sdks/java.png) +- [Apache Beam Python SDK](/documentation/sdks/python) ![Python logo](/images/logos/sdks/python.png) +- [Apache Beam Go SDK](/documentation/sdks/go) Go logo A Scala Scala logo interface is also available as [Scio](https://github.com/spotify/scio). @@ -41,14 +41,16 @@ A Scala Scala logo The Beam Pipeline Runners translate the data processing pipeline you define with your Beam program into the API compatible with the distributed processing back-end of your choice. When you run your Beam program, you'll need to specify an [appropriate runner](/documentation/runners/capability-matrix) for the back-end where you want to execute your pipeline. -Beam currently supports Runners that work with the following distributed processing back-ends: +Beam currently supports the following runners: -- Apache Flink ![Apache Flink logo](/images/logos/runners/flink.png) -- Apache Samza Apache Samza logo -- Apache Spark ![Apache Spark logo](/images/logos/runners/spark.png) -- Google Cloud Dataflow ![Google Cloud Dataflow logo](/images/logos/runners/dataflow.png) -- Hazelcast Jet ![Hazelcast Jet logo](/images/logos/runners/jet.png) -- Twister2 ![Twister2 logo](/images/logos/runners/twister2.png) +- [Direct Runner](/documentation/runners/direct) +- [Apache Flink Runner](/documentation/runners/flink) ![Apache Flink logo](/images/logos/runners/flink.png) +- [Apache Nemo Runner](/documentation/runners/nemo) +- [Apache Samza Runner](/documentation/runners/samza) Apache Samza logo +- [Apache Spark Runner](/documentation/runners/spark) ![Apache Spark logo](/images/logos/runners/spark.png) +- [Google Cloud Dataflow Runner](/documentation/runners/dataflow) ![Google Cloud Dataflow logo](/images/logos/runners/dataflow.png) +- [Hazelcast Jet Runner](/documentation/runners/jet) ![Hazelcast Jet logo](/images/logos/runners/jet.png) +- [Twister2 Runner](/documentation/runners/twister2) ![Twister2 logo](/images/logos/runners/twister2.png) **Note:** You can always execute your pipeline locally for testing and debugging purposes. @@ -56,9 +58,12 @@ Beam currently supports Runners that work with the following distributed process Get started using Beam for your data processing tasks. +> If you already know [Apache Spark](http://spark.apache.org/), +> check our [Getting started from Apache Spark](/get-started/from-spark) page. + 1. [Try Apache Beam](/get-started/try-apache-beam) in an online interactive environment. -1. Follow the Quickstart for the [Java SDK](/get-started/quickstart-java), the [Python SDK](/get-started/quickstart-py) or the [Go SDK](/get-started/quickstart-go). +1. Follow the Quickstart for the [Java SDK](/get-started/quickstart-java), the [Python SDK](/get-started/quickstart-py), or the [Go SDK](/get-started/quickstart-go). 1. See the [WordCount Examples Walkthrough](/get-started/wordcount-example) for examples that introduce various features of the SDKs. diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index 722fa1af98a1..dd482c288e46 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -88,18 +88,34 @@ versions denoted `0.x.y`. ## Releases +### 2.27.0 (2020-12-22) +Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.27.0/apache-beam-2.27.0-source-release.zip). +[SHA-512](https://downloads.apache.org/beam/2.27.0/apache-beam-2.27.0-source-release.zip.sha512). +[signature](https://downloads.apache.org/beam/2.27.0/apache-beam-2.27.0-source-release.zip.asc). + +[Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12349380). +[Blog post](/blog/beam-2.27.0). + +### 2.26.0 (2020-12-11) +Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.26.0/apache-beam-2.26.0-source-release.zip). +[SHA-512](https://downloads.apache.org/beam/2.26.0/apache-beam-2.26.0-source-release.zip.sha512). +[signature](https://downloads.apache.org/beam/2.26.0/apache-beam-2.26.0-source-release.zip.asc). + +[Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12348833). +[Blog post](/blog/beam-2.26.0). + ### 2.25.0 (2020-10-23) -Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.25.0/apache-beam-2.25.0-source-release.zip). -[SHA-512](https://downloads.apache.org/beam/2.25.0/apache-beam-2.25.0-source-release.zip.sha512). -[signature](https://downloads.apache.org/beam/2.25.0/apache-beam-2.25.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.25.0/apache-beam-2.25.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.25.0/apache-beam-2.25.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/.25.0/apache-beam-2.25.0-source-release.zip.asc). [Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347147). [Blog post](/blog/beam-2.25.0). ### 2.24.0 (2020-09-18) -Official [source code download](http://www.apache.org/dyn/closer.cgi/beam/2.24.0/apache-beam-2.24.0-source-release.zip). -[SHA-512](https://downloads.apache.org/beam/2.24.0/apache-beam-2.24.0-source-release.zip.sha512). -[signature](https://downloads.apache.org/beam/2.24.0/apache-beam-2.24.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.24.0/apache-beam-2.24.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.24.0/apache-beam-2.24.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.24.0/apache-beam-2.24.0-source-release.zip.asc). [Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347146). [Blog post](/blog/beam-2.24.0). @@ -123,7 +139,7 @@ Official [source code download](https://archive.apache.org/dist/beam/2.22.0/apac ### 2.21.0 (2020-05-27) Official [source code download](https://archive.apache.org/dist/beam/2.21.0/apache-beam-2.21.0-source-release.zip). [SHA-512](https://archive.apache.org/dist/beam/2.21.0/apache-beam-2.21.0-source-release.zip.sha512). -[signature](https://archive.apache.org/dist/2.21.0/apache-beam-2.21.0-source-release.zip.asc). +[signature](https://archive.apache.org/dist/beam/2.21.0/apache-beam-2.21.0-source-release.zip.asc). [Release notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527&version=12347143). [Blog post](/blog/beam-2.21.0). diff --git a/website/www/site/content/en/get-started/quickstart-java.md b/website/www/site/content/en/get-started/quickstart-java.md index 6c7f9a37dc24..4286a4de63f3 100644 --- a/website/www/site/content/en/get-started/quickstart-java.md +++ b/website/www/site/content/en/get-started/quickstart-java.md @@ -21,7 +21,7 @@ limitations under the License. # Apache Beam Java SDK Quickstart -This Quickstart will walk you through executing your first Beam pipeline to run [WordCount](/get-started/wordcount-example), written using Beam's [Java SDK](/documentation/sdks/java), on a [runner](/documentation#runners) of your choice. +This quickstart shows you how to set up a Java development environment and run an [example pipeline](/get-started/wordcount-example) written with the [Apache Beam Java SDK](/documentation/sdks/java), using a [runner](/documentation#runners) of your choice. If you're interested in contributing to the Apache Beam Java codebase, see the [Contribution Guide](/contribute). @@ -111,9 +111,11 @@ Ensure you are in the same directory as the `pom.xml` file generated from the pr $ gradle init {{< /highlight >}} +You'll be asked if you want to generate a Gradle build. Enter **yes**. You'll also be prompted to choose a DSL (Groovy or Kotlin). This tutorial uses Groovy, so select that if you don't have a preference. + After you have converted the project to Gradle: -1. Edit the generated `build.gradle` file by adding `mavenCentral()` under `repositories`: +1. In the generated `build.gradle` file, in the `repositories` block, replace `mavenLocal()` with `mavenCentral()`: {{< highlight >}} repositories { mavenCentral() @@ -140,18 +142,20 @@ task execute (type:JavaExec) { $ gradle build {{< /highlight >}} -## Run WordCount +## Run a pipeline -A single Beam pipeline can run on multiple Beam [runners](/documentation#runners), including the [FlinkRunner](/documentation/runners/flink), [SparkRunner](/documentation/runners/spark), [NemoRunner](/documentation/runners/nemo), [JetRunner](/documentation/runners/jet), or [DataflowRunner](/documentation/runners/dataflow). The [DirectRunner](/documentation/runners/direct) is a common runner for getting started, as it runs locally on your machine and requires no specific setup. +A single Beam pipeline can run on multiple Beam [runners](/documentation#runners), including the [FlinkRunner](/documentation/runners/flink), [SparkRunner](/documentation/runners/spark), [NemoRunner](/documentation/runners/nemo), [JetRunner](/documentation/runners/jet), or [DataflowRunner](/documentation/runners/dataflow). The [DirectRunner](/documentation/runners/direct) is a common runner for getting started, as it runs locally on your machine and requires no specific setup. If you're just trying out Beam and you're not sure what to use, use the [DirectRunner](/documentation/runners/direct). -After you've chosen which runner you'd like to use: +The general process for running a pipeline goes like this: 1. Ensure you've done any runner-specific setup. -1. Build your commandline by: - 1. Specifying a specific runner with `--runner=` (defaults to the [DirectRunner](/documentation/runners/direct)) - 1. Adding any runner-specific required options - 1. Choosing input files and an output location are accessible on the chosen runner. (For example, you can't access a local file if you are running the pipeline on an external cluster.) -1. Run your first WordCount pipeline. +1. Build your command line: + 1. Specify a runner with `--runner=` (defaults to the [DirectRunner](/documentation/runners/direct)). + 1. Add any runner-specific required options. + 1. Choose input files and an output location that are accessible to the runner. (For example, you can't access a local file if you are running the pipeline on an external cluster.) +1. Run the command. + +To run the WordCount pipeline, see the Maven and Gradle examples below. ### Run WordCount Using Maven From ca4c6a312fd2820205324763ffb129db374c063a Mon Sep 17 00:00:00 2001 From: Jakub Sadowski Date: Mon, 1 Feb 2021 12:17:56 +0100 Subject: [PATCH 18/18] examples update --- examples/java/build.gradle | 28 ++- .../apache/beam/examples/complete/README.md | 5 + .../kafkatopubsub/KafkaPubsubConstants.java | 29 +++ .../complete/kafkatopubsub/KafkaToPubsub.java | 237 ++++++++++++++++++ .../examples/complete/kafkatopubsub/README.md | 200 +++++++++++++++ .../kafkatopubsub/avro/AvroDataClass.java | 63 +++++ .../AvroDataClassKafkaAvroDeserializer.java | 41 +++ .../kafkatopubsub/avro/package-info.java | 20 ++ .../kafka/consumer/SslConsumerFactoryFn.java | 135 ++++++++++ .../kafkatopubsub/kafka/consumer/Utils.java | 173 +++++++++++++ .../kafka/consumer/package-info.java | 20 ++ .../options/KafkaToPubsubOptions.java | 96 +++++++ .../kafkatopubsub/options/package-info.java | 20 ++ .../complete/kafkatopubsub/package-info.java | 20 ++ .../transforms/FormatTransform.java | 128 ++++++++++ .../transforms/package-info.java | 20 ++ ...ryReadFromQueryWithBigQueryStorageAPI.java | 74 ++++++ .../complete/game/LeaderBoardTest.java | 2 +- .../kafkatopubsub/KafkaToPubsubE2ETest.java | 129 ++++++++++ .../kafkatopubsub/KafkaToPubsubTest.java | 90 +++++++ .../cookbook/CombinePerKeyExamplesTest.java | 13 +- examples/kotlin/build.gradle | 16 +- .../beam/examples/kotlin/MinimalWordCount.kt | 2 +- 23 files changed, 1540 insertions(+), 21 deletions(-) create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaPubsubConstants.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/README.md create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClass.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClassKafkaAvroDeserializer.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/package-info.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/SslConsumerFactoryFn.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/Utils.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/package-info.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/KafkaToPubsubOptions.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/package-info.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/package-info.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/package-info.java create mode 100644 examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java create mode 100644 examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java create mode 100644 examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubTest.java diff --git a/examples/java/build.gradle b/examples/java/build.gradle index c305f0451f09..5c057d897a02 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -18,8 +18,14 @@ import groovy.json.JsonOutput -plugins { id 'org.apache.beam.module' } +plugins { + id 'java' + id 'org.apache.beam.module' + id 'com.github.johnrengelman.shadow' +} + applyJavaNature( + enableStrictDependencies: true, exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples', ) @@ -49,9 +55,11 @@ configurations.sparkRunnerPreCommit { dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) compile library.java.vendored_guava_26_0_jre + compile library.java.kafka_clients compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:google-cloud-platform-core") compile project(":sdks:java:io:google-cloud-platform") + compile project(":sdks:java:io:kafka") compile project(":sdks:java:extensions:ml") compile library.java.avro compile library.java.bigdataoss_util @@ -63,17 +71,29 @@ dependencies { compile library.java.google_cloud_datastore_v1_proto_client compile library.java.google_code_gson compile library.java.google_http_client + compile library.java.google_oauth_client compile library.java.joda_time compile library.java.proto_google_cloud_datastore_v1 compile library.java.slf4j_api - compile library.java.slf4j_jdk14 - runtime project(path: ":runners:direct-java", configuration: "shadow") + compile library.java.vendored_grpc_1_26_0 + compile library.java.vendored_guava_26_0_jre + compile "com.google.api.grpc:proto-google-cloud-language-v1:1.81.4" + compile ("io.confluent:kafka-avro-serializer:5.3.2") { + // It depends on "spotbugs-annotations:3.1.9" which clashes with current + // "spotbugs-annotations:3.1.12" used in Beam. Not required. + exclude group: "org.apache.zookeeper", module: "zookeeper" + } + compile "org.apache.httpcomponents:httpclient:4.5.13" + compile "org.apache.httpcomponents:httpcore:4.4.13" + testCompile project(path: ":runners:direct-java", configuration: "shadow") testCompile project(":sdks:java:io:google-cloud-platform") testCompile project(":sdks:java:extensions:ml") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit testCompile library.java.mockito_core + testCompile library.java.testcontainers_kafka + testCompile library.java.testcontainers_gcloud // Add dependencies for the PreCommit configurations // For each runner a project level dependency on the examples project. @@ -82,7 +102,7 @@ dependencies { delegate.add(runner + "PreCommit", project(path: ":examples:java", configuration: "testRuntime")) } directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") - flinkRunnerPreCommit project(":runners:flink:1.10") + flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}") // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath // before 4.1.8.Final defined by Apache Beam sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/README.md b/examples/java/src/main/java/org/apache/beam/examples/complete/README.md index 3f4842a5c7a7..b74ea44554d5 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/README.md +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/README.md @@ -32,6 +32,11 @@ This directory contains end-to-end example pipelines that perform complex data p Pub/Sub topic, splits each line into individual words, capitalizes those words, and writes the output to a BigQuery table. +
  • KafkaToPubsub + — A streaming pipeline example that creates a pipeline to read data + from a single or multiple topics from Apache Kafka and write data into a single topic + in Google Cloud Pub/Sub. +
  • TfIdf — An example that computes a basic TF-IDF search table for a directory or Cloud Storage prefix. Demonstrates joining data, side inputs, and logging. diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaPubsubConstants.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaPubsubConstants.java new file mode 100644 index 000000000000..46f021a5ca75 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaPubsubConstants.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub; + +/** Constant variables that are used across the template's parts. */ +public class KafkaPubsubConstants { + + /* Config keywords */ + public static final String KAFKA_CREDENTIALS = "kafka"; + public static final String SSL_CREDENTIALS = "ssl"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + public static final String BUCKET = "bucket"; +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java new file mode 100644 index 000000000000..e4db72e2dc33 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub; + +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.configureKafka; +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.configureSsl; +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.getKafkaCredentialsFromVault; +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.isSslSpecified; +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.parseKafkaConsumerConfig; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.examples.complete.kafkatopubsub.avro.AvroDataClass; +import org.apache.beam.examples.complete.kafkatopubsub.avro.AvroDataClassKafkaAvroDeserializer; +import org.apache.beam.examples.complete.kafkatopubsub.options.KafkaToPubsubOptions; +import org.apache.beam.examples.complete.kafkatopubsub.transforms.FormatTransform; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Values; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The {@link KafkaToPubsub} pipeline is a streaming pipeline which ingests data in JSON format from + * Kafka, and outputs the resulting records to PubSub. Input topics, output topic, Bootstrap servers + * are specified by the user as template parameters.
    + * Kafka may be configured with SASL/SCRAM security mechanism, in this case a Vault secret storage + * with credentials should be provided. URL to credentials and Vault token are specified by the user + * as template parameters. + * + *

    Pipeline Requirements + * + *

      + *
    • Kafka Bootstrap Server(s). + *
    • Kafka Topic(s) exists. + *
    • The PubSub output topic exists. + *
    • (Optional) An existing HashiCorp Vault secret storage + *
    • (Optional) A configured secure SSL connection for Kafka + *
    + * + *

    Example Usage + * + * 

    + * # Gradle preparation
+ *
+ * To run this example your {@code build.gradle} file should contain the following task
+ * to execute the pipeline:
+ * {@code
+ * task execute (type:JavaExec) {
+ *     main = System.getProperty("mainClass")
+ *     classpath = sourceSets.main.runtimeClasspath
+ *     systemProperties System.getProperties()
+ *     args System.getProperty("exec.args", "").split()
+ * }
+ * }
+ *
+ * This task allows to run the pipeline via the following command:
+ * {@code
+ * gradle clean execute -DmainClass=org.apache.beam.examples.complete.kafkatopubsub.KafkaToPubsub \
+ *      -Dexec.args="--= --="
+ * }
+ *
+ * # Running the pipeline
+ * To execute this pipeline, specify the parameters:
+ *
+ * - Kafka Bootstrap servers
+ * - Kafka input topics
+ * - Pub/Sub output topic
+ * - Output format
+ *
+ * in the following format:
+ * {@code
+ * --bootstrapServers=host:port \
+ * --inputTopics=your-input-topic \
+ * --outputTopic=projects/your-project-id/topics/your-topic-pame \
+ * --outputFormat=AVRO|PUBSUB
+ * }
+ *
+ * Optionally, to retrieve Kafka credentials for SASL/SCRAM,
+ * specify a URL to the credentials in HashiCorp Vault and the vault access token:
+ * {@code
+ * --secretStoreUrl=http(s)://host:port/path/to/credentials
+ * --vaultToken=your-token
+ * }
+ *
+ * Optionally, to configure secure SSL connection between the Beam pipeline and Kafka,
+ * specify the parameters:
+ * - A path to a truststore file (it can be a local path or a GCS path, which should start with `gs://`)
+ * - A path to a keystore file (it can be a local path or a GCS path, which should start with `gs://`)
+ * - Truststore password
+ * - Keystore password
+ * - Key password
+ * {@code
+ * --truststorePath=path/to/kafka.truststore.jks
+ * --keystorePath=path/to/kafka.keystore.jks
+ * --truststorePassword=your-truststore-password
+ * --keystorePassword=your-keystore-password
+ * --keyPassword=your-key-password
+ * }
+ * By default this will run the pipeline locally with the DirectRunner. To change the runner, specify:
+ * {@code
+ * --runner=YOUR_SELECTED_RUNNER
+ * }
+ *
    + * + *

    Example Avro usage + * + * 

    + * This template contains an example Class to deserialize AVRO from Kafka and serialize it to AVRO in Pub/Sub.
+ *
+ * To use this example in the specific case, follow the few steps:
+ * 
      
+ * 
    •  Create your own class to describe AVRO schema. As an example use {@link AvroDataClass}. Just define necessary fields.
+ * 
    •  Create your own Avro Deserializer class. As an example use {@link AvroDataClassKafkaAvroDeserializer}. Just rename it, and put your own Schema class as the necessary types.
+ * 
    •  Modify the {@link FormatTransform}. Put your Schema class and Deserializer to the related parameter.
+ * 
    •  Modify write step in the {@link KafkaToPubsub} by put your Schema class to "writeAvrosToPubSub" step.
+ * 
    
+ *
    + */ +public class KafkaToPubsub { + + /* Logger for class.*/ + private static final Logger LOG = LoggerFactory.getLogger(KafkaToPubsub.class); + + /** + * Main entry point for pipeline execution. + * + * @param args Command line arguments to the pipeline. + */ + public static void main(String[] args) { + KafkaToPubsubOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(KafkaToPubsubOptions.class); + + // Create the pipeline + Pipeline pipeline = Pipeline.create(options); + run(pipeline, options); + } + + /** + * Runs a pipeline which reads message from Kafka and writes it to GCS. + * + * @param options arguments to the pipeline + */ + public static PipelineResult run(Pipeline pipeline, KafkaToPubsubOptions options) { + // Configure Kafka consumer properties + Map kafkaConfig = new HashMap<>(); + kafkaConfig.putAll(parseKafkaConsumerConfig(options.getKafkaConsumerConfig())); + Map sslConfig = new HashMap<>(); + if (options.getSecretStoreUrl() != null && options.getVaultToken() != null) { + Map> credentials = + getKafkaCredentialsFromVault(options.getSecretStoreUrl(), options.getVaultToken()); + kafkaConfig = configureKafka(credentials.get(KafkaPubsubConstants.KAFKA_CREDENTIALS)); + } else { + LOG.warn( + "No information to retrieve Kafka credentials was provided. " + + "Trying to initiate an unauthorized connection."); + } + + if (isSslSpecified(options)) { + sslConfig.putAll(configureSsl(options)); + } else { + LOG.info( + "No information to retrieve SSL certificate was provided by parameters." + + "Trying to initiate a plain text connection."); + } + + List topicsList = new ArrayList<>(Arrays.asList(options.getInputTopics().split(","))); + + checkArgument( + topicsList.size() > 0 && topicsList.get(0).length() > 0, + "inputTopics cannot be an empty string."); + + List bootstrapServersList = + new ArrayList<>(Arrays.asList(options.getBootstrapServers().split(","))); + + checkArgument( + bootstrapServersList.size() > 0 && topicsList.get(0).length() > 0, + "bootstrapServers cannot be an empty string."); + + LOG.info( + "Starting Kafka-To-PubSub pipeline with parameters bootstrap servers:" + + options.getBootstrapServers() + + " input topics: " + + options.getInputTopics() + + " output pubsub topic: " + + options.getOutputTopic()); + + /* + * Steps: + * 1) Read messages in from Kafka + * 2) Extract values only + * 3) Write successful records to PubSub + */ + + if (options.getOutputFormat() == FormatTransform.FORMAT.AVRO) { + pipeline + .apply( + "readAvrosFromKafka", + FormatTransform.readAvrosFromKafka( + options.getBootstrapServers(), topicsList, kafkaConfig, sslConfig)) + .apply("createValues", Values.create()) + .apply("writeAvrosToPubSub", PubsubIO.writeAvros(AvroDataClass.class)); + + } else { + pipeline + .apply( + "readFromKafka", + FormatTransform.readFromKafka( + options.getBootstrapServers(), topicsList, kafkaConfig, sslConfig)) + .apply("createValues", Values.create()) + .apply("writeToPubSub", new FormatTransform.FormatOutput(options)); + } + + return pipeline.run(); + } +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/README.md b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/README.md new file mode 100644 index 000000000000..13f2d3a5f593 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/README.md @@ -0,0 +1,200 @@ + + +# Apache Beam pipeline example to ingest data from Apache Kafka to Google Cloud Pub/Sub + +This directory contains an [Apache Beam](https://beam.apache.org/) pipeline example that creates a pipeline to read data +from a single or multiple topics from +[Apache Kafka](https://kafka.apache.org/) and write data into a single topic +in [Google Cloud Pub/Sub](https://cloud.google.com/pubsub). + +Supported data formats: + +- Serializable plaintext formats, such as JSON +- [PubSubMessage](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage). + +Supported input source configurations: + +- Single or multiple Apache Kafka bootstrap servers +- Apache Kafka SASL/SCRAM authentication over plaintext or SSL connection +- Secrets vault service [HashiCorp Vault](https://www.vaultproject.io/). + +Supported destination configuration: + +- Single Google Cloud Pub/Sub topic. + +In a simple scenario, the example will create an Apache Beam pipeline that will read messages from a source Kafka server +with a source topic, and stream the text messages into specified Pub/Sub destination topic. Other scenarios may need +Kafka SASL/SCRAM authentication, that can be performed over plain text or SSL encrypted connection. The example supports +using a single Kafka user account to authenticate in the provided source Kafka servers and topics. To support SASL +authenticaton over SSL the example will need an SSL certificate location and access to a secrets vault service with +Kafka username and password, currently supporting HashiCorp Vault. + +## Requirements + +- Java 8 +- Kafka Bootstrap Server(s) up and running +- Existing source Kafka topic(s) +- An existing Pub/Sub destination output topic +- (Optional) An existing HashiCorp Vault +- (Optional) A configured secure SSL connection for Kafka + +## Getting Started + +This section describes what is needed to get the example up and running. + +- Gradle preparation +- Local execution +- Running as a Dataflow Template +- Supported Output Formats + - PubSubMessage + - AVRO +- E2E tests (TBD) + +## Gradle preparation + +To run this example your `build.gradle` file should contain the following task to execute the pipeline: + +``` +task execute (type:JavaExec) { + main = System.getProperty("mainClass") + classpath = sourceSets.main.runtimeClasspath + systemProperties System.getProperties() + args System.getProperty("exec.args", "").split() +} +``` + +This task allows to run the pipeline via the following command: + +```bash +gradle clean execute -DmainClass=org.apache.beam.examples.complete.kafkatopubsub.KafkaToPubsub \ + -Dexec.args="--= --=" +``` + +## Running the pipeline + +To execute this pipeline, specify the parameters: + +- Kafka Bootstrap servers +- Kafka input topics +- Pub/Sub output topic +- Output format + +in the following format: + +```bash +--bootstrapServers=host:port \ +--inputTopics=your-input-topic \ +--outputTopic=projects/your-project-id/topics/your-topic-pame \ +--outputFormat=AVRO|PUBSUB +``` + +Optionally, to retrieve Kafka credentials for SASL/SCRAM, specify a URL to the credentials in HashiCorp Vault and the +vault access token: + +```bash +--secretStoreUrl=http(s)://host:port/path/to/credentials +--vaultToken=your-token +``` + +Optionally, to configure secure SSL connection between the Beam pipeline and Kafka, specify the parameters: + +- A path to a truststore file (it can be a local path or a GCS path, which should start with `gs://`) +- A path to a keystore file (it can be a local path or a GCS path, which should start with `gs://`) +- Truststore password +- Keystore password +- Key password + +```bash +--truststorePath=path/to/kafka.truststore.jks +--keystorePath=path/to/kafka.keystore.jks +--truststorePassword=your-truststore-password +--keystorePassword=your-keystore-password +--keyPassword=your-key-password +``` + +By default this will run the pipeline locally with the DirectRunner. To change the runner, specify: + +```bash +--runner=YOUR_SELECTED_RUNNER +``` + +See the [documentation](http://beam.apache.org/get-started/quickstart/) and +the [Examples README](../../../../../../../../../README.md) for more information about how to run this example. + +## Running as a Dataflow Template + +This example also exists as Google Dataflow Template, which you can build and run using Google Cloud Platform. See +its [README.md](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/master/v2/kafka-to-pubsub/README.md) for +more information. + +## Supported Output Formats + +This pipeline can output data in a format of PubSubMessage or AVRO. + +### PubSubMessage + +This example supports PubSubMessage format for output out-of-the-box. No additional changes are required. + +### AVRO + +This example contains an example demonstrating AVRO format support, the following steps should be done to provide it: + +- Define custom Class to deserialize AVRO from Kafka [provided in example] +- Create custom data serialization in Apache Beam +- Serialize data to AVRO in Pub/Sub [provided in example]. + +To use this example in the specific case, follow these steps: + +- Create your own class to describe AVRO schema. As an example use [AvroDataClass](avro/AvroDataClass.java). Just define + necessary fields. +- Create your own Avro Deserializer class. As an example + use [AvroDataClassKafkaAvroDeserializer class](avro/AvroDataClassKafkaAvroDeserializer.java). Just rename it, and put + your own Schema class as the necessary types. +- Modify the [FormatTransform.readAvrosFromKafka method](transforms/FormatTransform.java). Put your Schema class and + Deserializer to the related parameter. + +```java +return KafkaIO.read() + ... + .withValueDeserializerAndCoder( + AvroDataClassKafkaAvroDeserializer.class,AvroCoder.of(AvroDataClass.class)) // put your classes here + ... +``` + +- [OPTIONAL TO IMPLEMENT] Add [Beam Transform](https://beam.apache.org/documentation/programming-guide/#transforms) if + it necessary in your case. +- Modify the write step in the [KafkaToPubsub class](KafkaToPubsub.java) by putting your Schema class to " + writeAvrosToPubSub" step. + - NOTE: if it changed during the transform, you should use changed one class definition. + +```java +if(options.getOutputFormat()==FormatTransform.FORMAT.AVRO){ + ... + .apply("writeAvrosToPubSub",PubsubIO.writeAvros(AvroDataClass.class)); // put your SCHEMA class here + + } +``` + +## End to end tests + +TBD + +_Note: The Kafka to Pub/Sub job executed with a distributed runner supports SSL configuration with the certificate +located only in GCS._ diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClass.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClass.java new file mode 100644 index 000000000000..8c8702115f65 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClass.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.avro; + +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.DefaultCoder; + +/** + * Example of AVRO serialization class. To configure your AVRO schema, change this class to + * requirement schema definition + */ +@DefaultCoder(AvroCoder.class) +public class AvroDataClass { + + String field1; + Float field2; + Float field3; + + public AvroDataClass(String field1, Float field2, Float field3) { + this.field1 = field1; + this.field2 = field2; + this.field3 = field3; + } + + public String getField1() { + return field1; + } + + public void setField1(String field1) { + this.field1 = field1; + } + + public Float getField2() { + return field2; + } + + public void setField2(Float field2) { + this.field2 = field2; + } + + public Float getField3() { + return field3; + } + + public void setField3(Float field3) { + this.field3 = field3; + } +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClassKafkaAvroDeserializer.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClassKafkaAvroDeserializer.java new file mode 100644 index 000000000000..a9aeb72eb196 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/AvroDataClassKafkaAvroDeserializer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.avro; + +import io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer; +import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; +import java.util.Map; +import org.apache.kafka.common.serialization.Deserializer; + +/** Example of custom AVRO Deserialize. */ +public class AvroDataClassKafkaAvroDeserializer extends AbstractKafkaAvroDeserializer + implements Deserializer { + + @Override + public void configure(Map configs, boolean isKey) { + configure(new KafkaAvroDeserializerConfig(configs)); + } + + @Override + public AvroDataClass deserialize(String s, byte[] bytes) { + return (AvroDataClass) this.deserialize(bytes); + } + + @Override + public void close() {} +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/package-info.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/package-info.java new file mode 100644 index 000000000000..1dc4f36ce656 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/avro/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Kafka to Pubsub template. */ +package org.apache.beam.examples.complete.kafkatopubsub.avro; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/SslConsumerFactoryFn.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/SslConsumerFactoryFn.java new file mode 100644 index 000000000000..eda4e9e14f8b --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/SslConsumerFactoryFn.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.beam.sdk.io.FileSystems; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Class to create Kafka Consumer with configured SSL. */ +public class SslConsumerFactoryFn + implements SerializableFunction, Consumer> { + private final Map sslConfig; + private static final String TRUSTSTORE_LOCAL_PATH = "/tmp/kafka.truststore.jks"; + private static final String KEYSTORE_LOCAL_PATH = "/tmp/kafka.keystore.jks"; + + /* Logger for class.*/ + private static final Logger LOG = LoggerFactory.getLogger(SslConsumerFactoryFn.class); + + public SslConsumerFactoryFn(Map sslConfig) { + this.sslConfig = sslConfig; + } + + @SuppressWarnings("nullness") + @Override + public Consumer apply(Map config) { + String truststoreLocation = sslConfig.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG); + String keystoreLocation = sslConfig.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG); + if (truststoreLocation == null || keystoreLocation == null) { + LOG.warn("Not enough information to configure SSL"); + return new KafkaConsumer<>(config); + } + + try { + if (truststoreLocation.startsWith("gs://")) { + getGcsFileAsLocal(truststoreLocation, TRUSTSTORE_LOCAL_PATH); + sslConfig.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, TRUSTSTORE_LOCAL_PATH); + } else { + checkFileExists(truststoreLocation); + } + + if (keystoreLocation.startsWith("gs://")) { + getGcsFileAsLocal(keystoreLocation, KEYSTORE_LOCAL_PATH); + sslConfig.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, KEYSTORE_LOCAL_PATH); + } else { + checkFileExists(keystoreLocation); + } + } catch (IOException e) { + LOG.error("Failed to retrieve data for SSL", e); + return new KafkaConsumer<>(config); + } + + config.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.name()); + config.put( + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, + sslConfig.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)); + config.put( + SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, + sslConfig.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)); + config.put( + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, + sslConfig.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)); + config.put( + SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, + sslConfig.get(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)); + config.put( + SslConfigs.SSL_KEY_PASSWORD_CONFIG, sslConfig.get(SslConfigs.SSL_KEY_PASSWORD_CONFIG)); + + return new KafkaConsumer<>(config); + } + + private void checkFileExists(String filePath) throws IOException { + LOG.info( + "Trying to get file: {} locally. Local files don't support when in using distribute runner", + filePath); + File f = new File(filePath); + if (f.exists()) { + LOG.debug("{} exists", f.getAbsolutePath()); + } else { + LOG.error("{} does not exist", f.getAbsolutePath()); + throw new IOException(); + } + } + + /** + * Reads a file from GCS and writes it locally. + * + * @param gcsFilePath path to file in GCS in format "gs://your-bucket/path/to/file" + * @param outputFilePath path where to save file locally + * @throws IOException thrown if not able to read or write file + */ + public static void getGcsFileAsLocal(String gcsFilePath, String outputFilePath) + throws IOException { + LOG.info("Reading contents from GCS file: {}", gcsFilePath); + Set options = new HashSet<>(2); + options.add(StandardOpenOption.CREATE); + options.add(StandardOpenOption.APPEND); + // Copy the GCS file into a local file and will throw an I/O exception in case file not found. + try (ReadableByteChannel readerChannel = + FileSystems.open(FileSystems.matchSingleFileSpec(gcsFilePath).resourceId())) { + try (FileChannel writeChannel = FileChannel.open(Paths.get(outputFilePath), options)) { + writeChannel.transferFrom(readerChannel, 0, Long.MAX_VALUE); + } + } + } +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/Utils.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/Utils.java new file mode 100644 index 000000000000..c99fec5efe93 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/Utils.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer; + +import static org.apache.beam.examples.complete.kafkatopubsub.KafkaPubsubConstants.KAFKA_CREDENTIALS; +import static org.apache.beam.examples.complete.kafkatopubsub.KafkaPubsubConstants.PASSWORD; +import static org.apache.beam.examples.complete.kafkatopubsub.KafkaPubsubConstants.USERNAME; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.beam.examples.complete.kafkatopubsub.options.KafkaToPubsubOptions; +import org.apache.beam.vendor.grpc.v1p26p0.com.google.gson.JsonObject; +import org.apache.beam.vendor.grpc.v1p26p0.com.google.gson.JsonParser; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.security.scram.internals.ScramMechanism; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Utilities for construction of Kafka Consumer. */ +public class Utils { + + /* Logger for class.*/ + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + /** + * Retrieves all credentials from HashiCorp Vault secret storage. + * + * @param secretStoreUrl url to the secret storage that contains a credentials for Kafka + * @param token Vault token to access the secret storage + * @return credentials for Kafka consumer config + */ + public static Map> getKafkaCredentialsFromVault( + String secretStoreUrl, String token) { + Map> credentialMap = new HashMap<>(); + + JsonObject credentials = null; + try { + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(secretStoreUrl); + request.addHeader("X-Vault-Token", token); + HttpResponse response = client.execute(request); + String json = EntityUtils.toString(response.getEntity(), "UTF-8"); + + /* + Vault's response JSON has a specific schema, where the actual data is placed under + {data: {data: }}. + Example: + { + "request_id": "6a0bb14b-ef24-256c-3edf-cfd52ad1d60d", + "lease_id": "", + "renewable": false, + "lease_duration": 0, + "data": { + "data": { + "bucket": "kafka_to_pubsub_test", + "key_password": "secret", + "keystore_password": "secret", + "keystore_path": "ssl_cert/kafka.keystore.jks", + "password": "admin-secret", + "truststore_password": "secret", + "truststore_path": "ssl_cert/kafka.truststore.jks", + "username": "admin" + }, + "metadata": { + "created_time": "2020-10-20T11:43:11.109186969Z", + "deletion_time": "", + "destroyed": false, + "version": 8 + } + }, + "wrap_info": null, + "warnings": null, + "auth": null + } + */ + // Parse security properties from the response JSON + credentials = + JsonParser.parseString(json) + .getAsJsonObject() + .get("data") + .getAsJsonObject() + .getAsJsonObject("data"); + } catch (IOException e) { + LOG.error("Failed to retrieve credentials from Vault.", e); + } + + if (credentials != null) { + // Username and password for Kafka authorization + credentialMap.put(KAFKA_CREDENTIALS, new HashMap<>()); + + if (credentials.has(USERNAME) && credentials.has(PASSWORD)) { + credentialMap.get(KAFKA_CREDENTIALS).put(USERNAME, credentials.get(USERNAME).getAsString()); + credentialMap.get(KAFKA_CREDENTIALS).put(PASSWORD, credentials.get(PASSWORD).getAsString()); + } else { + LOG.warn( + "There are no username and/or password for Kafka in Vault." + + "Trying to initiate an unauthorized connection."); + } + } + + return credentialMap; + } + + /** + * Configures Kafka consumer for authorized connection. + * + * @param props username and password for Kafka + * @return configuration set of parameters for Kafka + */ + public static Map configureKafka(@Nullable Map props) { + // Create the configuration for Kafka + Map config = new HashMap<>(); + if (props != null && props.containsKey(USERNAME) && props.containsKey(PASSWORD)) { + config.put(SaslConfigs.SASL_MECHANISM, ScramMechanism.SCRAM_SHA_512.mechanismName()); + config.put( + SaslConfigs.SASL_JAAS_CONFIG, + String.format( + "org.apache.kafka.common.security.scram.ScramLoginModule required " + + "username=\"%s\" password=\"%s\";", + props.get(USERNAME), props.get(PASSWORD))); + } + return config; + } + + public static Map configureSsl(KafkaToPubsubOptions options) { + Map config = new HashMap<>(); + config.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, options.getTruststorePath()); + config.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, options.getKeystorePath()); + config.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, options.getTruststorePassword()); + config.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, options.getKeystorePassword()); + config.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, options.getKeyPassword()); + + return config; + } + + public static boolean isSslSpecified(KafkaToPubsubOptions options) { + return options.getTruststorePath() != null + || options.getTruststorePassword() != null + || options.getKeystorePath() != null + || options.getKeyPassword() != null; + } + + public static Map parseKafkaConsumerConfig(String kafkaConsumerConfig) { + return Arrays.stream(kafkaConsumerConfig.split(";")) + .map(s -> s.split("=")) + .collect(Collectors.toMap(kv -> kv[0], kv -> kv[1])); + } +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/package-info.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/package-info.java new file mode 100644 index 000000000000..54aff5e766e7 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Kafka to Pubsub template. */ +package org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/KafkaToPubsubOptions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/KafkaToPubsubOptions.java new file mode 100644 index 000000000000..3a86b7f4345d --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/KafkaToPubsubOptions.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.options; + +import org.apache.beam.examples.complete.kafkatopubsub.transforms.FormatTransform; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.Validation; + +public interface KafkaToPubsubOptions extends PipelineOptions { + @Description( + "Comma Separated list of Kafka Bootstrap Servers (e.g: server1:[port],server2:[port]).") + @Validation.Required + String getBootstrapServers(); + + void setBootstrapServers(String value); + + @Description( + "Comma Separated list of Kafka topic(s) to read the input from (e.g: topic1,topic2).") + @Validation.Required + String getInputTopics(); + + void setInputTopics(String value); + + @Description( + "The Cloud Pub/Sub topic to publish to. " + + "The name should be in the format of " + + "projects//topics/.") + @Validation.Required + String getOutputTopic(); + + void setOutputTopic(String outputTopic); + + @Description( + "Format which will be writen to output Pub/Sub topic. Supported formats: AVRO, PUBSUB") + @Validation.Required + FormatTransform.FORMAT getOutputFormat(); + + void setOutputFormat(FormatTransform.FORMAT outputFormat); + + @Description("URL to credentials in Vault") + String getSecretStoreUrl(); + + void setSecretStoreUrl(String secretStoreUrl); + + @Description("Vault token") + String getVaultToken(); + + void setVaultToken(String vaultToken); + + @Description("The path to the trust store file") + String getTruststorePath(); + + void setTruststorePath(String truststorePath); + + @Description("The path to the key store file") + String getKeystorePath(); + + void setKeystorePath(String keystorePath); + + @Description("The password for the trust store password") + String getTruststorePassword(); + + void setTruststorePassword(String truststorePassword); + + @Description("The store password for the key store password") + String getKeystorePassword(); + + void setKeystorePassword(String keystorePassword); + + @Description("The password of the private key in the key store file") + String getKeyPassword(); + + void setKeyPassword(String keyPassword); + + @Description( + "Additional kafka consumer configs to be applied to Kafka Consumer (e.g. key1=value1;key2=value2).") + String getKafkaConsumerConfig(); + + void setKafkaConsumerConfig(String kafkaConfig); +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/package-info.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/package-info.java new file mode 100644 index 000000000000..361a5eab058c --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/options/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Kafka to Pubsub template. */ +package org.apache.beam.examples.complete.kafkatopubsub.options; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/package-info.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/package-info.java new file mode 100644 index 000000000000..66b4c04f763f --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Kafka to Pubsub template. */ +package org.apache.beam.examples.complete.kafkatopubsub; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java new file mode 100644 index 000000000000..d493c0648180 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub.transforms; + +import java.util.List; +import java.util.Map; +import org.apache.beam.examples.complete.kafkatopubsub.avro.AvroDataClass; +import org.apache.beam.examples.complete.kafkatopubsub.avro.AvroDataClassKafkaAvroDeserializer; +import org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.SslConsumerFactoryFn; +import org.apache.beam.examples.complete.kafkatopubsub.options.KafkaToPubsubOptions; +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.NullableCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage; +import org.apache.beam.sdk.io.kafka.KafkaIO; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.kafka.common.serialization.StringDeserializer; + +/** Different transformations over the processed data in the pipeline. */ +public class FormatTransform { + + public enum FORMAT { + PUBSUB, + AVRO + } + + /** + * Configures Kafka consumer. + * + * @param bootstrapServers Kafka servers to read from + * @param topicsList Kafka topics to read from + * @param kafkaConfig configuration for the Kafka consumer + * @param sslConfig configuration for the SSL connection + * @return configured reading from Kafka + */ + public static PTransform>> readFromKafka( + String bootstrapServers, + List topicsList, + Map kafkaConfig, + Map sslConfig) { + return KafkaIO.read() + .withBootstrapServers(bootstrapServers) + .withTopics(topicsList) + .withKeyDeserializerAndCoder( + StringDeserializer.class, (Coder) NullableCoder.of(StringUtf8Coder.of())) + .withValueDeserializerAndCoder( + StringDeserializer.class, (Coder) NullableCoder.of(StringUtf8Coder.of())) + .withConsumerConfigUpdates(kafkaConfig) + .withConsumerFactoryFn(new SslConsumerFactoryFn(sslConfig)) + .withoutMetadata(); + } + + /** + * Configures Kafka consumer to read avros to {@link AvroDataClass} format. + * + * @param bootstrapServers Kafka servers to read from + * @param topicsList Kafka topics to read from + * @param config configuration for the Kafka consumer + * @return configured reading from Kafka + */ + public static PTransform>> readAvrosFromKafka( + String bootstrapServers, + List topicsList, + Map config, + Map sslConfig) { + return KafkaIO.read() + .withBootstrapServers(bootstrapServers) + .withTopics(topicsList) + .withKeyDeserializerAndCoder( + StringDeserializer.class, (Coder) NullableCoder.of(StringUtf8Coder.of())) + .withValueDeserializerAndCoder( + AvroDataClassKafkaAvroDeserializer.class, AvroCoder.of(AvroDataClass.class)) + .withConsumerConfigUpdates(config) + .withConsumerFactoryFn(new SslConsumerFactoryFn(sslConfig)) + .withoutMetadata(); + } + + /** + * The {@link FormatOutput} wraps a String serializable messages with the {@link PubsubMessage} + * class. + */ + public static class FormatOutput extends PTransform, PDone> { + + private final KafkaToPubsubOptions options; + + public FormatOutput(KafkaToPubsubOptions options) { + this.options = options; + } + + @Override + public PDone expand(PCollection input) { + return input + .apply( + "convertMessagesToPubsubMessages", + MapElements.into(TypeDescriptor.of(PubsubMessage.class)) + .via( + (String json) -> + new PubsubMessage(json.getBytes(Charsets.UTF_8), ImmutableMap.of()))) + .apply( + "writePubsubMessagesToPubSub", PubsubIO.writeMessages().to(options.getOutputTopic())); + } + } +} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/package-info.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/package-info.java new file mode 100644 index 000000000000..83b9d5c3c956 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Kafka to Pubsub template. */ +package org.apache.beam.examples.complete.kafkatopubsub.transforms; diff --git a/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java new file mode 100644 index 000000000000..22680ac76809 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.snippets.transforms.io.gcp.bigquery; + +// [START bigquery_read_from_query_with_bigquery_storage_api] + +import org.apache.beam.examples.snippets.transforms.io.gcp.bigquery.BigQueryMyData.MyData; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; + +class BigQueryReadFromQueryWithBigQueryStorageAPI { + public static PCollection readFromQueryWithBigQueryStorageAPI( + String project, String dataset, String table, String query, Pipeline pipeline) { + + // String project = "my-project-id"; + // String dataset = "my_bigquery_dataset_id"; + // String table = "my_bigquery_table_id"; + + // Pipeline pipeline = Pipeline.create(); + + /* + String query = String.format("SELECT\n" + + " string_field,\n" + + " int64_field,\n" + + " float64_field,\n" + + " numeric_field,\n" + + " bool_field,\n" + + " bytes_field,\n" + + " date_field,\n" + + " datetime_field,\n" + + " time_field,\n" + + " timestamp_field,\n" + + " geography_field,\n" + + " array_field,\n" + + " struct_field\n" + + "FROM\n" + + " `%s:%s.%s`", project, dataset, table) + */ + + PCollection rows = + pipeline + .apply( + "Read from BigQuery table", + BigQueryIO.readTableRows() + .fromQuery(query) + .usingStandardSql() + .withMethod(Method.DIRECT_READ)) + .apply( + "TableRows to MyData", + MapElements.into(TypeDescriptor.of(MyData.class)).via(MyData::fromTableRow)); + + return rows; + } +} +// [END bigquery_read_from_query_with_bigquery_storage_api] diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java index 4632887953e4..e48c2471cea7 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java @@ -17,8 +17,8 @@ */ package org.apache.beam.examples.complete.game; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; -import static org.junit.Assert.assertThat; import java.io.Serializable; import org.apache.beam.examples.complete.game.LeaderBoard.CalculateTeamScores; diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java new file mode 100644 index 000000000000..8e44fc2258dd --- /dev/null +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasProperty; + +import com.google.auth.Credentials; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.apache.beam.examples.complete.kafkatopubsub.options.KafkaToPubsubOptions; +import org.apache.beam.examples.complete.kafkatopubsub.transforms.FormatTransform.FORMAT; +import org.apache.beam.runners.direct.DirectOptions; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.extensions.gcp.auth.NoopCredentialFactory; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubOptions; +import org.apache.beam.sdk.io.gcp.pubsub.TestPubsub; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.containers.PubSubEmulatorContainer; +import org.testcontainers.utility.DockerImageName; + +/** E2E test for {@link KafkaToPubsub} pipeline. */ +public class KafkaToPubsubE2ETest { + private static final String PUBSUB_EMULATOR_IMAGE = + "gcr.io/google.com/cloudsdktool/cloud-sdk:316.0.0-emulators"; + private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:5.4.3"; + private static final String PUBSUB_MESSAGE = "test pubsub message"; + private static final String KAFKA_TOPIC_NAME = "messages-topic"; + private static final String PROJECT_ID = "try-kafka-pubsub"; + private static final PipelineOptions OPTIONS = TestPipeline.testingPipelineOptions(); + + @ClassRule + public static final PubSubEmulatorContainer PUB_SUB_EMULATOR_CONTAINER = + new PubSubEmulatorContainer(DockerImageName.parse(PUBSUB_EMULATOR_IMAGE)); + + @ClassRule + public static final KafkaContainer KAFKA_CONTAINER = + new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE_NAME)); + + @Rule public final transient TestPipeline pipeline = TestPipeline.fromOptions(OPTIONS); + @Rule public final transient TestPubsub testPubsub = TestPubsub.fromOptions(OPTIONS); + + @BeforeClass + public static void beforeClass() throws Exception { + Credentials credentials = NoopCredentialFactory.fromOptions(OPTIONS).getCredential(); + OPTIONS.as(DirectOptions.class).setBlockOnRun(false); + OPTIONS.as(GcpOptions.class).setGcpCredential(credentials); + OPTIONS.as(GcpOptions.class).setProject(PROJECT_ID); + OPTIONS + .as(PubsubOptions.class) + .setPubsubRootUrl("http://" + PUB_SUB_EMULATOR_CONTAINER.getEmulatorEndpoint()); + OPTIONS.as(KafkaToPubsubOptions.class).setOutputFormat(FORMAT.PUBSUB); + OPTIONS + .as(KafkaToPubsubOptions.class) + .setBootstrapServers(KAFKA_CONTAINER.getBootstrapServers()); + OPTIONS.as(KafkaToPubsubOptions.class).setInputTopics(KAFKA_TOPIC_NAME); + OPTIONS + .as(KafkaToPubsubOptions.class) + .setKafkaConsumerConfig(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG + "=earliest"); + } + + @Before + public void setUp() { + OPTIONS.as(KafkaToPubsubOptions.class).setOutputTopic(testPubsub.topicPath().getPath()); + } + + @Test + public void testKafkaToPubsubE2E() throws Exception { + PipelineResult job = KafkaToPubsub.run(pipeline, OPTIONS.as(KafkaToPubsubOptions.class)); + + sendKafkaMessage(); + testPubsub + .assertThatTopicEventuallyReceives( + hasProperty("payload", equalTo(PUBSUB_MESSAGE.getBytes(StandardCharsets.UTF_8)))) + .waitForUpTo(Duration.standardMinutes(1)); + try { + job.cancel(); + } catch (UnsupportedOperationException e) { + throw new AssertionError("Could not stop pipeline.", e); + } + } + + private void sendKafkaMessage() { + try (KafkaProducer producer = + new KafkaProducer<>( + ImmutableMap.of( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + KAFKA_CONTAINER.getBootstrapServers(), + ProducerConfig.CLIENT_ID_CONFIG, + UUID.randomUUID().toString()), + new StringSerializer(), + new StringSerializer())) { + producer.send(new ProducerRecord<>(KAFKA_TOPIC_NAME, "testcontainers", PUBSUB_MESSAGE)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Something went wrong in kafka producer", e); + } + } +} diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubTest.java new file mode 100644 index 000000000000..e71b30459723 --- /dev/null +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.complete.kafkatopubsub; + +import static org.apache.beam.examples.complete.kafkatopubsub.KafkaPubsubConstants.PASSWORD; +import static org.apache.beam.examples.complete.kafkatopubsub.KafkaPubsubConstants.USERNAME; +import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.getKafkaCredentialsFromVault; + +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.security.scram.internals.ScramMechanism; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Test of KafkaToPubsub. */ +@RunWith(JUnit4.class) +public class KafkaToPubsubTest { + + /** Tests configureKafka() with a null input properties. */ + @Test + public void testConfigureKafkaNullProps() { + Map config = Utils.configureKafka(null); + Assert.assertEquals(new HashMap<>(), config); + } + + /** Tests configureKafka() without a Password in input properties. */ + @Test + public void testConfigureKafkaNoPassword() { + Map props = new HashMap<>(); + props.put(USERNAME, "username"); + Map config = Utils.configureKafka(props); + Assert.assertEquals(new HashMap<>(), config); + } + + /** Tests configureKafka() without a Username in input properties. */ + @Test + public void testConfigureKafkaNoUsername() { + Map props = new HashMap<>(); + props.put(PASSWORD, "password"); + Map config = Utils.configureKafka(props); + Assert.assertEquals(new HashMap<>(), config); + } + + /** Tests configureKafka() with an appropriate input properties. */ + @Test + public void testConfigureKafka() { + Map props = new HashMap<>(); + props.put(USERNAME, "username"); + props.put(PASSWORD, "password"); + + Map expectedConfig = new HashMap<>(); + expectedConfig.put(SaslConfigs.SASL_MECHANISM, ScramMechanism.SCRAM_SHA_512.mechanismName()); + expectedConfig.put( + SaslConfigs.SASL_JAAS_CONFIG, + String.format( + "org.apache.kafka.common.security.scram.ScramLoginModule required " + + "username=\"%s\" password=\"%s\";", + props.get(USERNAME), props.get(PASSWORD))); + + Map config = Utils.configureKafka(props); + Assert.assertEquals(expectedConfig, config); + } + + /** Tests getKafkaCredentialsFromVault() with an invalid url. */ + @Test + public void testGetKafkaCredentialsFromVaultInvalidUrl() { + Map> credentials = + getKafkaCredentialsFromVault("some-url", "some-token"); + Assert.assertEquals(new HashMap<>(), credentials); + } +} diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/CombinePerKeyExamplesTest.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/CombinePerKeyExamplesTest.java index eca98c3b9f8c..cd19bf993684 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/CombinePerKeyExamplesTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/CombinePerKeyExamplesTest.java @@ -17,6 +17,8 @@ */ package org.apache.beam.examples.cookbook; +import static org.hamcrest.MatcherAssert.assertThat; + import com.google.api.services.bigquery.model.TableRow; import java.util.List; import org.apache.beam.examples.cookbook.CombinePerKeyExamples.ExtractLargeWordsFn; @@ -24,7 +26,6 @@ import org.apache.beam.sdk.transforms.DoFnTester; import org.apache.beam.sdk.values.KV; import org.hamcrest.CoreMatchers; -import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -69,9 +70,9 @@ public void testExtractLargeWordsFn() throws Exception { DoFnTester> extractLargeWordsFn = DoFnTester.of(new ExtractLargeWordsFn()); List> results = extractLargeWordsFn.processBundle(ROWS_ARRAY); - Assert.assertThat(results, CoreMatchers.hasItem(tuple1)); - Assert.assertThat(results, CoreMatchers.hasItem(tuple2)); - Assert.assertThat(results, CoreMatchers.hasItem(tuple3)); + assertThat(results, CoreMatchers.hasItem(tuple1)); + assertThat(results, CoreMatchers.hasItem(tuple2)); + assertThat(results, CoreMatchers.hasItem(tuple3)); } @Test @@ -79,7 +80,7 @@ public void testFormatShakespeareOutputFn() throws Exception { DoFnTester, TableRow> formatShakespeareOutputFn = DoFnTester.of(new FormatShakespeareOutputFn()); List results = formatShakespeareOutputFn.processBundle(COMBINED_TUPLES_ARRAY); - Assert.assertThat(results, CoreMatchers.hasItem(resultRow1)); - Assert.assertThat(results, CoreMatchers.hasItem(resultRow2)); + assertThat(results, CoreMatchers.hasItem(resultRow1)); + assertThat(results, CoreMatchers.hasItem(resultRow2)); } } diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index e2525298d3a2..9999f42174c6 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -22,7 +22,7 @@ plugins { id 'org.apache.beam.module' id 'org.jetbrains.kotlin.jvm' version '1.3.72' } -applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples.kotlin') +applyJavaNature(enableStrictDependencies: true, exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples.kotlin') provideIntegrationTestingDependencies() enableJavaPerformanceTesting() @@ -46,6 +46,8 @@ configurations.sparkRunnerPreCommit { exclude group: "org.slf4j", module: "slf4j-jdk14" } +def kotlin_version = "1.3.72" + dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) compile library.java.vendored_guava_26_0_jre @@ -59,18 +61,15 @@ dependencies { compile library.java.google_api_services_pubsub compile library.java.google_auth_library_credentials compile library.java.google_auth_library_oauth2_http - compile library.java.google_cloud_datastore_v1_proto_client compile library.java.google_http_client compile library.java.joda_time - compile library.java.proto_google_cloud_datastore_v1 compile library.java.slf4j_api - compile library.java.slf4j_jdk14 + compile library.java.guava + compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + compile "org.jetbrains:annotations:13.0" runtime project(path: ":runners:direct-java", configuration: "shadow") testCompile project(":sdks:java:io:google-cloud-platform") - testCompile library.java.hamcrest_core - testCompile library.java.hamcrest_library testCompile library.java.junit - testCompile library.java.mockito_core // Add dependencies for the PreCommit configurations // For each runner a project level dependency on the examples project. @@ -79,7 +78,7 @@ dependencies { delegate.add(runner + "PreCommit", project(path: ":examples:kotlin", configuration: "testRuntime")) } directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") - flinkRunnerPreCommit project(":runners:flink:1.10") + flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}") // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath // before 4.1.8.Final defined by Apache Beam sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" @@ -87,7 +86,6 @@ dependencies { sparkRunnerPreCommit project(":sdks:java:io:hadoop-file-system") sparkRunnerPreCommit library.java.spark_streaming sparkRunnerPreCommit library.java.spark_core - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" } /* diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/MinimalWordCount.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/MinimalWordCount.kt index 2c752c4565c2..b95a495f00b4 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/MinimalWordCount.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/MinimalWordCount.kt @@ -96,7 +96,7 @@ public object MinimalWordCount { // individual word in Shakespeare's collected texts. .apply( FlatMapElements.into(TypeDescriptors.strings()) - .via(ProcessFunction> { input -> input.split("[^\\p{L}]+").toList() }) + .via(ProcessFunction> { input -> input.split("[^\\p{L}]+".toRegex()).toList() }) ) // We use a Filter transform to avoid empty word .apply(Filter.by(SerializableFunction { input -> !input.isEmpty() }))