diff --git a/website/www/site/assets/icons/contributor/become a committer/beam-logo-icon.svg b/website/www/site/assets/icons/contributor/become a committer/beam-logo-icon.svg new file mode 100644 index 000000000000..587667d29cc9 --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/beam-logo-icon.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/contributor/become a committer/code-icon.svg b/website/www/site/assets/icons/contributor/become a committer/code-icon.svg new file mode 100644 index 000000000000..338acf8e8de1 --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/code-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/contributor/become a committer/diamond-icon.svg b/website/www/site/assets/icons/contributor/become a committer/diamond-icon.svg new file mode 100644 index 000000000000..50930d56bf94 --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/diamond-icon.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/contributor/become a committer/file-icon.svg b/website/www/site/assets/icons/contributor/become a committer/file-icon.svg new file mode 100644 index 000000000000..a198565b15a4 --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/file-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/contributor/become a committer/messages-icon.svg b/website/www/site/assets/icons/contributor/become a committer/messages-icon.svg new file mode 100644 index 000000000000..0f710d68e9f6 --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/messages-icon.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + diff --git a/website/www/site/assets/icons/contributor/become a committer/tool-icon.svg b/website/www/site/assets/icons/contributor/become a committer/tool-icon.svg new file mode 100644 index 000000000000..46e30c82ab1a --- /dev/null +++ b/website/www/site/assets/icons/contributor/become a committer/tool-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/scss/_global.sass b/website/www/site/assets/scss/_global.sass index fbc02977971b..773b4048840a 100644 --- a/website/www/site/assets/scss/_global.sass +++ b/website/www/site/assets/scss/_global.sass @@ -144,8 +144,8 @@ body color: #8c8b8e .without_switcher border-top-left-radius: 8px - pre - background: initial !important + pre + background: initial !important table margin-top: 24px diff --git a/website/www/site/assets/scss/_lists.scss b/website/www/site/assets/scss/_lists.scss index b175081e1e25..5e7587711f90 100644 --- a/website/www/site/assets/scss/_lists.scss +++ b/website/www/site/assets/scss/_lists.scss @@ -56,7 +56,7 @@ .arrow-list-header { color: #333333; - text-decoration: none; + text-decoration: none !important; display: flex; max-height: 40px; margin-bottom: 74px; @@ -284,6 +284,11 @@ height: auto !important; visibility: visible; } + .collapsing { + position:relative; + height:unset !important; + overflow:hidden; + } } } .beam-list { @@ -299,4 +304,3 @@ height: 44px; } } - diff --git a/website/www/site/assets/scss/_navbar-desktop.scss b/website/www/site/assets/scss/_navbar-desktop.scss index d3eb4e9832f3..826fcbe97e83 100644 --- a/website/www/site/assets/scss/_navbar-desktop.scss +++ b/website/www/site/assets/scss/_navbar-desktop.scss @@ -15,182 +15,177 @@ * limitations under the License. */ -@import "media"; - -.navigation-bar-mobile { - display: none; - .arrow-icon { - display: flex; - align-items: baseline; - margin-left: 10px; - } -} -.nav-tabs { - border-bottom: none; -} -.navigation-bar-desktop { - display: flex; - height: 96px; - width: 100%; - align-items: center; - margin-bottom: 30px; - box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); - background-color: $color-white; - z-index: 10000; // just to make sure that navbar always on top of other elements - - #iconsBar { - display: flex; - } - - a { - @extend .component-text; - font-weight: 500; - color: $color-dark-gray; - letter-spacing: 0.2px; - line-height: 1.63; - margin-right: 39px; - text-decoration: none; - cursor: pointer; - } - - .navbar-logo { - margin-left: 58px; - margin-bottom: 4px; - img { - width: 88px; - } - } - .navbar-bar-left { - display: flex; - justify-content: space-between; - width: 100%; - } - .navbar-links { - display: flex; - align-items: center; - justify-content: space-between; - z-index: 10000; - :last-child { - margin-right: 0; - } - - .navbar-link { - display: inline-block; - position: relative; - margin-bottom: 2px; - } - - .navbar-link::before { - transition: 0.3; - content: ""; - position: absolute; - background-color: $color-sun; - height: 0%; - width: 100%; - bottom: 0px; - border-radius: 5px; - } - - .navbar-link:hover::before { - height: 2px; - } - - .navbar-dropdown-documentation { - list-style-type: none; - - .dropdown-toggle { - margin: 0; - } - - ul { - width: 209px; - left: -25%; - text-align: center; - border: none; - box-shadow: none; - padding-top: 34px; - padding-bottom: 0; - border-radius: 0; - li { - height: 36px; - } - a { - @extend .component-text; - line-height: 1.63; - letter-spacing: 0.2px; - } - } - } - } - - .navbar-dropdown-apache { - margin-right: 90px; - list-style-type: none; - - .dropdown-toggle { - margin: 0; - } - - ul { - width: 209px; - left: 70%; - transform: translateX(-50%); - text-align: center; - border: none; - box-shadow: none; - padding-top: 35px; - padding-bottom: 0; - border-radius: 0; - li { - height: 36px; - } - a { - @extend .component-text; - - margin-right: 0 !important; - } - } - - .arrow-icon { - position: absolute; - top: 3px; - right: -30px; - } - } - - .navbar-dropdown-apache:hover { - .arrow-icon { - opacity: 0.84; - } - } - - .navbar-dropdown { - .dropdown-menu > li > a { - &:hover, - &:focus { - text-decoration: none; - color: $color-dropdown-link-hover-text; - background-color: $color-dropdown-link-hover-bg; - } - } - } - - .dropdown:hover .dropdown-menu { - display: block; - margin-top: 0; - } -} - -@media (max-width: $tablet) { - .navigation-bar-desktop { - display: none; - } - - .navigation-bar-mobile { - display: block; - } - - .page-nav { - margin-top: 30px; - } -} + @import "media"; + + .navigation-bar-mobile { + display: none; + .arrow-icon { + display: flex; + align-items: baseline; + margin-left: 10px; + } + } + .nav-tabs { + border-bottom: none; + } + .navigation-bar-desktop { + display: flex; + height: 96px; + width: 100%; + align-items: center; + margin-bottom: 30px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + z-index: 10000; // just to make sure that navbar always on top of other elements + + #iconsBar { + display: flex; + } + + a { + @extend .component-text; + font-weight: 500; + color: $color-dark-gray; + letter-spacing: 0.2px; + line-height: 1.63; + margin-right: 39px; + text-decoration: none; + cursor: pointer; + } + + .navbar-logo { + margin-left: 58px; + margin-bottom: 4px; + img { + width: 88px; + } + } + .navbar-bar-left { + display: flex; + justify-content: space-between; + width: 100%; + } + .navbar-links { + display: flex; + align-items: center; + justify-content: space-between; + z-index: 10000; + :last-child { + margin-right: 0; + } + + .navbar-link { + display: inline-block; + position: relative; + margin-bottom: 2px; + } + + .navbar-link::before { + transition: 0.3; + content: ""; + position: absolute; + background-color: $color-sun; + height: 0%; + width: 100%; + bottom: 0px; + border-radius: 5px; + } + + .navbar-link:hover::before { + height: 2px; + } + + .navbar-dropdown-documentation { + list-style-type: none; + ul { + width: 209px; + left: -25%; + text-align: center; + border: none; + box-shadow: none; + padding-top: 34px; + padding-bottom: 0; + border-radius: 0; + li { + height: 36px; + } + a { + @extend .component-text; + line-height: 1.63; + letter-spacing: 0.2px; + } + } + } + } + + .navbar-dropdown-apache { + margin-right: 90px; + list-style-type: none; + + .dropdown-toggle { + margin: 0; + } + + ul { + width: 209px; + left: 70%; + transform: translateX(-50%); + text-align: center; + border: none; + box-shadow: none; + padding-top: 35px; + padding-bottom: 0; + border-radius: 0; + li { + height: 36px; + } + a { + @extend .component-text; + + margin-right: 0 !important; + } + } + + .arrow-icon { + position: absolute; + top: 3px; + right: -30px; + } + } + + .navbar-dropdown-apache:hover { + .arrow-icon { + opacity: 0.84; + } + } + + .navbar-dropdown { + .dropdown-menu > li > a { + &:hover, + &:focus { + text-decoration: none; + color: $color-dropdown-link-hover-text; + background-color: $color-dropdown-link-hover-bg; + } + } + } + + .dropdown:hover .dropdown-menu { + display: block; + margin-top: 0; + } + } + + @media (max-width: $tablet) { + .navigation-bar-desktop { + display: none; + } + + .navigation-bar-mobile { + display: block; + } + + .page-nav { + margin-top: 30px; + } + } diff --git a/website/www/site/assets/scss/_syntax-highlighting.scss b/website/www/site/assets/scss/_syntax-highlighting.scss index ed9f27c32c9a..2d12caf37be6 100644 --- a/website/www/site/assets/scss/_syntax-highlighting.scss +++ b/website/www/site/assets/scss/_syntax-highlighting.scss @@ -16,71 +16,203 @@ * Syntax highlighting styles */ .highlight { - background: #fff; + //background: #fff; - .chroma { - background: #eef; - } + .chroma { + background: #eef; + } - .c { color: #998; font-style: italic } // Comment - .err { color: #a61717 } // Error - .k { font-weight: bold } // Keyword - .o { font-weight: bold } // Operator - .cm { color: #998; font-style: italic } // Comment.Multiline - .cp { color: #999; font-weight: bold } // Comment.Preproc - .c1 { color: #998; font-style: italic } // Comment.Single - .cs { color: #999; font-weight: bold; font-style: italic } // Comment.Special - .gd { color: #000; background-color: #fdd } // Generic.Deleted - .gd .x { color: #000; background-color: #faa } // Generic.Deleted.Specific - .ge { font-style: italic } // Generic.Emph - .gr { color: #a00 } // Generic.Error - .gh { color: #999 } // Generic.Heading - .gi { color: #000; background-color: #dfd } // Generic.Inserted - .gi .x { color: #000; background-color: #afa } // Generic.Inserted.Specific - .go { color: #888 } // Generic.Output - .gp { color: #555 } // Generic.Prompt - .gs { font-weight: bold } // Generic.Strong - .gu { color: #aaa } // Generic.Subheading - .gt { color: #a00 } // Generic.Traceback - .kc { font-weight: bold } // Keyword.Constant - .kd { font-weight: bold } // Keyword.Declaration - .kp { font-weight: bold } // Keyword.Pseudo - .kr { font-weight: bold } // Keyword.Reserved - .kt { color: #458; font-weight: bold } // Keyword.Type - .m { color: #099 } // Literal.Number - .s { color: #d14 } // Literal.String - .na { color: #008080 } // Name.Attribute - .nb { color: #0086B3 } // Name.Builtin - .nc { color: #458; font-weight: bold } // Name.Class - .no { color: #008080 } // Name.Constant - .ni { color: #800080 } // Name.Entity - .ne { color: #900; font-weight: bold } // Name.Exception - .nf { color: #900; font-weight: bold } // Name.Function - .nn { color: #555 } // Name.Namespace - .nt { color: #000080 } // Name.Tag - .nv { color: #008080 } // Name.Variable - .ow { font-weight: bold } // Operator.Word - .w { color: #bbb } // Text.Whitespace - .mf { color: #099 } // Literal.Number.Float - .mh { color: #099 } // Literal.Number.Hex - .mi { color: #099 } // Literal.Number.Integer - .mo { color: #099 } // Literal.Number.Oct - .sb { color: #d14 } // Literal.String.Backtick - .sc { color: #d14 } // Literal.String.Char - .sd { color: #d14 } // Literal.String.Doc - .s2 { color: #d14 } // Literal.String.Double - .se { color: #d14 } // Literal.String.Escape - .sh { color: #d14 } // Literal.String.Heredoc - .si { color: #d14 } // Literal.String.Interpol - .sx { color: #d14 } // Literal.String.Other - .sr { color: #009926 } // Literal.String.Regex - .s1 { color: #d14 } // Literal.String.Single - .ss { color: #990073 } // Literal.String.Symbol - .bp { color: #999 } // Name.Builtin.Pseudo - .vc { color: #008080 } // Name.Variable.Class - .vg { color: #008080 } // Name.Variable.Global - .vi { color: #008080 } // Name.Variable.Instance - .il { color: #099 } // Literal.Number.Integer.Long + .c { + color: #998; + font-style: italic; + } // Comment + .err { + color: #a61717; + } // Error + .k { + font-weight: bold; + } // Keyword + .o { + font-weight: bold; + } // Operator + .cm { + color: #998; + font-style: italic; + } // Comment.Multiline + .cp { + color: #999; + font-weight: bold; + } // Comment.Preproc + .c1 { + color: #998; + font-style: italic; + } // Comment.Single + .cs { + color: #999; + font-weight: bold; + font-style: italic; + } // Comment.Special + .gd { + color: #000; + background-color: #fdd; + } // Generic.Deleted + .gd .x { + color: #000; + background-color: #faa; + } // Generic.Deleted.Specific + .ge { + font-style: italic; + } // Generic.Emph + .gr { + color: #a00; + } // Generic.Error + .gh { + color: #999; + } // Generic.Heading + .gi { + color: #000; + background-color: #dfd; + } // Generic.Inserted + .gi .x { + color: #000; + background-color: #afa; + } // Generic.Inserted.Specific + .go { + color: #888; + } // Generic.Output + .gp { + color: #555; + } // Generic.Prompt + .gs { + font-weight: bold; + } // Generic.Strong + .gu { + color: #aaa; + } // Generic.Subheading + .gt { + color: #a00; + } // Generic.Traceback + .kc { + font-weight: bold; + } // Keyword.Constant + .kd { + font-weight: bold; + } // Keyword.Declaration + .kp { + font-weight: bold; + } // Keyword.Pseudo + .kr { + font-weight: bold; + } // Keyword.Reserved + .kt { + color: #458; + font-weight: bold; + } // Keyword.Type + .m { + color: #099; + } // Literal.Number + .s { + color: #d14; + } // Literal.String + .na { + color: #008080; + } // Name.Attribute + .nb { + color: #0086b3; + } // Name.Builtin + .nc { + color: #458; + font-weight: bold; + } // Name.Class + .no { + color: #008080; + } // Name.Constant + .ni { + color: #800080; + } // Name.Entity + .ne { + color: #900; + font-weight: bold; + } // Name.Exception + .nf { + color: #900; + font-weight: bold; + } // Name.Function + .nn { + color: #555; + } // Name.Namespace + .nt { + color: #000080; + } // Name.Tag + .nv { + color: #008080; + } // Name.Variable + .ow { + font-weight: bold; + } // Operator.Word + .w { + color: #bbb; + } // Text.Whitespace + .mf { + color: #099; + } // Literal.Number.Float + .mh { + color: #099; + } // Literal.Number.Hex + .mi { + color: #099; + } // Literal.Number.Integer + .mo { + color: #099; + } // Literal.Number.Oct + .sb { + color: #d14; + } // Literal.String.Backtick + .sc { + color: #d14; + } // Literal.String.Char + .sd { + color: #d14; + } // Literal.String.Doc + .s2 { + color: #d14; + } // Literal.String.Double + .se { + color: #d14; + } // Literal.String.Escape + .sh { + color: #d14; + } // Literal.String.Heredoc + .si { + color: #d14; + } // Literal.String.Interpol + .sx { + color: #d14; + } // Literal.String.Other + .sr { + color: #009926; + } // Literal.String.Regex + .s1 { + color: #d14; + } // Literal.String.Single + .ss { + color: #990073; + } // Literal.String.Symbol + .bp { + color: #999; + } // Name.Builtin.Pseudo + .vc { + color: #008080; + } // Name.Variable.Class + .vg { + color: #008080; + } // Name.Variable.Global + .vi { + color: #008080; + } // Name.Variable.Instance + .il { + color: #099; + } // Literal.Number.Integer.Long } .highlighter-custom { @@ -103,3 +235,124 @@ pre { background: #eef; } +[class*="-switcher"] { + ul { + li { + margin-bottom:0 !important; + padding-left: 0 !important; + &:hover { + cursor: pointer; + } + a { + border: none; + margin-right: 0; + color: #333; + text-decoration: none; + &:hover { + background-color: #f2804d; + color: #fff; + } + } + &::before { + display: none; + } + } + li.active { + color: #fff; + a { + background-color: #ff6d05; + color: #fff; + border: none; + } + } + } +} +.language-switcher { + ul.nav-tabs { + padding-left: 0; + border:0; + li { + &:hover { + cursor: pointer; + } + a { + border: none; + margin-right: 0; + color: #333; + &:hover { + background-color: #f2804d; + color: #fff; + } + } + &::before { + display: none; + } + } + li.active { + color: #fff; + a { + background-color: #ff6d05; + color: #fff; + border: none; + } + } + } +} +nav.version-switcher { + ul { + li { + &:hover { + cursor: pointer; + } + a { + border: none; + margin-right: 0; + color: #333; + &:hover { + background-color: #f2804d; + color: #fff; + } + } + &::before { + display: none; + } + } + li.active { + color: #fff; + a { + background-color: #ff6d05; + color: #fff; + border: none; + } + } + } +} +nav.runner-switcher { + ul { + li { + &:hover { + cursor: pointer; + } + a { + border: none; + margin-right: 0; + color: #333; + &:hover { + background-color: #f2804d; + color: #fff; + } + } + &::before { + display: none; + } + } + li.active { + color: #fff; + a { + background-color: #ff6d05; + color: #fff; + border: none; + } + } + } +} diff --git a/website/www/site/assets/scss/_toggler-nav.scss b/website/www/site/assets/scss/_toggler-nav.scss index 34ed4e81e1ce..0cc6ba4f2a61 100644 --- a/website/www/site/assets/scss/_toggler-nav.scss +++ b/website/www/site/assets/scss/_toggler-nav.scss @@ -26,7 +26,7 @@ nav.language-switcher { background-color: #f8f8f8; &.active { - background-color: #222c37; + background-color: #ff6d05; color: #fff; } } diff --git a/website/www/site/assets/scss/_traits.scss b/website/www/site/assets/scss/_traits.scss new file mode 100644 index 000000000000..3e74b8d9831d --- /dev/null +++ b/website/www/site/assets/scss/_traits.scss @@ -0,0 +1,49 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +@import "media"; +.row_of_traits { + display: flex; + justify-content: space-between; + flex-direction: row; + .traits-item { + display: flex; + flex-direction: column; + justify-content: flex-start; + align-items: center; + margin-top: 64px; + .traits-item-icon { + width: 112px; + height: 112px; + margin-bottom: 48px; + } + .traits-item-description { + max-width: 228px; + text-align: center; + font-size: 14px; + line-height: 1.57; + letter-spacing: .43px; + .font-weight-bold { + font-weight: 700; + } + a{ + color: #333333; + } + } + } +} +@media (max-width: $mobile) { + .row_of_traits { + flex-direction: column; + } +} diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index a4d8877f3199..f030747f70a5 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -15,187 +15,187 @@ * limitations under the License. */ -@import "media"; + @import "media"; -.component-title { - font-size: 36px; - font-weight: 500; - font-style: normal; - line-height: 1.1; - letter-spacing: normal; - margin: 0; - color: $color-gray; -} + .component-title { + font-size: 36px; + font-weight: 500; + font-style: normal; + line-height: 1.1; + letter-spacing: normal; + margin: 0; + color: $color-gray; + } -.component-large-header { - font-size: 30px; - font-weight: 500; - font-style: normal; - line-height: 1.4; - letter-spacing: normal; - margin: 0; - color: $color-gray; -} + .component-large-header { + font-size: 30px; + font-weight: 500; + font-style: normal; + line-height: 1.4; + letter-spacing: normal; + margin: 0; + color: $color-gray; + } -.component-header { - font-size: 22px; - font-weight: 500; - font-style: normal; - line-height: 1.45; - letter-spacing: 0.43px; - margin: 0; - color: $color-gray; -} + .component-header { + font-size: 22px; + font-weight: 500; + font-style: normal; + line-height: 1.45; + letter-spacing: 0.43px; + margin: 0; + color: $color-gray; + } -.component-small-header { - font-size: 18px; - font-weight: 500; - font-style: normal; - line-height: 1.56; - letter-spacing: 0.43px; - margin: 0; - color: $color-gray; -} + .component-small-header { + font-size: 18px; + font-weight: 500; + font-style: normal; + line-height: 1.56; + letter-spacing: 0.43px; + margin: 0; + color: $color-gray; + } -.component-text { - font-size: 16px; - font-weight: normal; - font-style: normal; - line-height: 1.63; - letter-spacing: 0.43px; - margin: 0; - color: $color-gray; -} + .component-text { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.63; + letter-spacing: 0.43px; + margin: 0; + color: $color-gray; + } -.component-label { - font-size: 16px; - font-weight: bold; - font-stretch: normal; - font-style: normal; - line-height: 1.5; - letter-spacing: 0.43px; - margin: 0; - color: $color-gray; -} + .component-label { + font-size: 16px; + font-weight: bold; + font-stretch: normal; + font-style: normal; + line-height: 1.5; + letter-spacing: 0.43px; + margin: 0; + color: $color-gray; + } -.component-tag { - font-size: 16px; - font-weight: normal; - font-style: normal; - line-height: 1.63; - letter-spacing: 0.43px; - margin: 0; - color: $color-smoke; -} + .component-tag { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.63; + letter-spacing: 0.43px; + margin: 0; + color: $color-smoke; + } -.component-quote { - font-size: 20px; - font-weight: normal; - font-stretch: normal; - font-style: italic; - line-height: 1.44; - letter-spacing: 0.43px; - text-align: center; - color: $color-gray; -} + .component-quote { + font-size: 20px; + font-weight: normal; + font-stretch: normal; + font-style: italic; + line-height: 1.44; + letter-spacing: 0.43px; + text-align: center; + color: $color-gray; + } -.hero-title { - font-size: 16px; - font-weight: normal; - font-style: normal; - line-height: 1.88; - letter-spacing: 0.8px; - color: $color-white; -} + .hero-title { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.88; + letter-spacing: 0.8px; + color: $color-white; + } -.hero-heading { - font-size: 46px; - font-weight: 500; - font-style: normal; - line-height: 1; - letter-spacing: normal; - color: $color-white; -} + .hero-heading { + font-size: 46px; + font-weight: 500; + font-style: normal; + line-height: 1; + letter-spacing: normal; + color: $color-white; + } -.hero-subheading { - font-size: 20px; - font-weight: normal; - font-style: normal; - line-height: 1.44; - letter-spacing: normal; - color: $color-white; -} + .hero-subheading { + font-size: 20px; + font-weight: normal; + font-style: normal; + line-height: 1.44; + letter-spacing: normal; + color: $color-white; + } -@media (max-width: $tablet) { - .component-title { - font-size: 28px; - line-height: 1.29; - } - .component-large-header { - font-size: 24px; - } -} + @media (max-width: $tablet) { + .component-title { + font-size: 28px; + line-height: 1.29; + } + .component-large-header { + font-size: 24px; + } + } -h1 { - font-size: 36px; - font-weight: 500; - line-height: 1.1; - letter-spacing: normal; - margin-bottom: 24px; - margin-top: 64px; -} -h2 { - font-size: 30px; - font-weight: 500; - line-height: 1.2; - letter-spacing: normal; - margin-bottom: 24px; - margin-top: 56px; -} -h3 { - font-size: 24px; - font-weight: 500; - line-height: 1.25; - letter-spacing: normal; - margin-bottom: 16px; - margin-top: 48px; -} -h4 { - font-size: 22px; - font-weight: 500; - line-height: 1.36; - letter-spacing: normal; - margin-bottom: 12px; - margin-top: 48px; -} -h5 { - font-size: 18px; - font-weight: 500; - line-height: 1.6; - letter-spacing: 0.43px; - margin-bottom: 12px; - margin-top: 32px; -} -h6 { - font-size: 16px; - font-weight: bold; - line-height: 1.2; - letter-spacing: 0.43px; - margin-bottom: 12px; - margin-top: 32px; -} + h1 { + font-size: 36px; + font-weight: 500; + line-height: 1.1; + letter-spacing: normal; + margin-bottom: 24px; + margin-top: 64px; + } + h2 { + font-size: 30px; + font-weight: 500; + line-height: 1.2; + letter-spacing: normal; + margin-bottom: 24px; + margin-top: 56px; + } + h3 { + font-size: 24px; + font-weight: 500; + line-height: 1.25; + letter-spacing: normal; + margin-bottom: 16px; + margin-top: 48px; + } + h4 { + font-size: 22px; + font-weight: 500; + line-height: 1.36; + letter-spacing: normal; + margin-bottom: 12px; + margin-top: 48px; + } + h5 { + font-size: 18px; + font-weight: 500; + line-height: 1.6; + letter-spacing: 0.43px; + margin-bottom: 12px; + margin-top: 32px; + } + h6 { + font-size: 16px; + font-weight: bold; + line-height: 1.2; + letter-spacing: 0.43px; + margin-bottom: 12px; + margin-top: 32px; + } -p { - font-size: 16px; - font-weight: normal; - line-height: 1.63; - letter-spacing: 0.43px; -} -.hero-heading { - font-size: 32px; -} -li { - font-size: 16px; - letter-spacing: 0.43px; - line-height: 1.63; -} + p { + font-size: 16px; + font-weight: normal; + line-height: 1.63; + letter-spacing: 0.43px; + } + .hero-heading { + font-size: 32px; + } + li { + font-size: 16px; + letter-spacing: 0.43px; + line-height: 1.63; + } diff --git a/website/www/site/assets/scss/capability-matrix.scss b/website/www/site/assets/scss/capability-matrix.scss index 6bd629ada529..d5e5fc348aae 100644 --- a/website/www/site/assets/scss/capability-matrix.scss +++ b/website/www/site/assets/scss/capability-matrix.scss @@ -11,131 +11,285 @@ See the License for the specific language governing permissions and limitations under the License. */ +@import "media"; -/* What/Where/When/How colors. */ -.wwwh-what-dark { - color:#ca1; - font-weight:bold; - font-style:italic; +.back-button { + font-size: 14px; + font-weight: 700; + padding-left: 15px; + line-height: 1.14; + letter-spacing: 0.6px; + color: #f26628; + text-transform: uppercase; + &:hover { + color: #f2804d; + } + i { + width: 26px; + } } -.wwwh-where-dark { - color:#37d; - font-weight:bold; - font-style:italic; -} +.information-container { + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + flex-direction: row; + margin-bottom: 84px; -.wwwh-when-dark { - color:#6a4; - font-weight:bold; - font-style:italic; + @media (max-width: $mobile) { + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + flex-direction: column; + } + table { + margin: 0; + } + .read-tables { + table-layout: fixed; + position: relative; + th:first-of-type { + width: 22px; + min-width: 22px; + } + span { + font-weight: 500; + } + } + .second-container { + width: 270px; + margin-left: 100px; + @media (max-width: $mobile) { + margin-left: 0; + } + h5 { + margin-bottom: 22px; + } + .row { + display: -webkit-box; + display: flex; + margin-left: 0; + } + .box { + width: 71px; + height: 24px; + margin-bottom: 10px; + text-align: center; + margin-right: 10px; + } + .white { + border: solid 1px #f6f6f6; + background-color: #fff; + } + .partial { + border: solid 1px #d8d8d8; + background-color: #f9f9f9; + } + .gray { + border: solid 1px #bcbcbc; + background-color: #e1e0e0; + } + } + .border-left { + border-left: 3px solid #ff6d05; + padding: 7px; + font-size: 12px; + font-weight: 500; + line-height: 1.83; + letter-spacing: 0.43px; + } + .border-right { + transform: rotate(270deg); + height: 80px; + text-align: center; + padding: 0 !important; + width: 25px !important; + font-size: 12px; + font-weight: 500; + font-stretch: normal; + font-style: normal; + line-height: 1.57; + letter-spacing: 0.43px; + position: absolute; + top: 55px; + left: 24px; + } + .border-top { + border-top: 3px solid #ff6d05; + font-size: 14px; + font-weight: 400; + font-stretch: normal; + font-style: normal; + line-height: 1.57; + letter-spacing: 0.43px; + td { + width: 162px; + padding: 7px; + } + } } - -.wwwh-how-dark { - color:#b55; - font-weight:bold; - font-style:italic; +.table-container { + display: -webkit-box; + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + flex-direction: row; + margin-bottom: 32px; + table { + margin: 0; + } + .big-right { + max-width: 1240px; + } + .table-right { + overflow-x: scroll; + padding: 1px; + scrollbar-width: none; + &::-webkit-scrollbar { + width: 0; + background: 0 0; + } + } + .table-headers { + overflow-x: scroll; + border-spacing: 0; + border: solid 1px #d8d8d8; + border-bottom: none; + mix-blend-mode: multiply; + -ms-overflow-style: none; + scrollbar-width: none; + &::-webkit-scrollbar { + width: 0; + background: 0 0; + } + table { + height: 48px; + table-layout: fixed; + width: 710px; + th { + text-align: center; + width: 142px; + font-size: 12px; + font-weight: 500; + line-height: normal; + letter-spacing: normal; + padding-left: 10px; + @media (max-width: $mobile) { + width: 73px; + } + } + } + } + .big-headers table th { + width: 249px; + } + .table-center { + overflow-x: scroll; + border-spacing: 0; + border: solid 1px #d8d8d8; + border-collapse: separate; + transform: rotateX(180deg); + padding-bottom: 8px; + scrollbar-color: #ff6d00; + &::-webkit-scrollbar { + height: 8px; + width: 25px !important; + } + &::-webkit-scrollbar-track { + background-color: #efeded; + background-clip: content-box; + } + &::-webkit-scrollbar-thumb { + border-radius: 8px; + height: 6px !important; + background-color: #ff6d00; + } + table { + transform: rotateX(180deg); + border-collapse: separate; + table-layout: fixed; + width: 710px; + th, + td { + width: 142px; + height: 48px; + font-size: 12px; + font-weight: 500; + line-height: normal; + letter-spacing: normal; + border: 1px solid #fff; + @media (max-width: 640px) { + width: 73px; + } + } + } + } + .big-center { + table { + tr { + vertical-align: baseline; + th, + td { + width: 249px; + height: 352px; + padding: 6px; + font-weight: 400; + p { + font-size: 12px; + font-weight: 500; + line-height: normal; + letter-spacing: normal; + margin: 0; + } + } + } + } + } + .table-left table { + width: 142px; + @media (max-width: 640px) { + width: 73px; + } + tr:first-child th { + height: 83px; + border: none; + @media (max-width: $mobile) { + height: 119px; + } + @-moz-document url-prefix() { + height: 77px; + } + } + th { + height: 48px; + font-size: 12px; + font-weight: 500; + line-height: normal; + letter-spacing: normal; + border: solid 1px #f6f6f6; + border-right: none; + padding: 6px; + } + } + .big-left table { + text-align: start; + tr { + vertical-align: baseline; + } + th { + height: 352px; + } + } } - -/* Capability matrix general sizing, alignment etc. */ -table.cap { - border-spacing: 0px; - border-collapse: collapse; - padding: 2px; -} - -td.cap { - border-width: 1px; - border-style:solid; - vertical-align:text-top; - padding:0.5ex; -} - -th.cap, tr.cap, table.cap { - border-width: 1px; - border-style:solid; - vertical-align:text-top; - padding:0.5ex; -} - -td.cap-blank { - padding:10px; -} - -/* Capability matrix blog-post sizing, alignment etc. */ -table.cap-summary { - border-spacing: 0px; - border-collapse: collapse; - padding: 2px; - width:600px; -} - -td.cap-summary { - border-width: 1px; - border-style:solid; - vertical-align:text-top; - padding:0.5ex; -} - -th.cap-summary, tr.cap-summary, table.cap-summary { - border-width: 1px; - border-style:solid; - vertical-align:text-top; - padding:0.5ex; -} - -td.cap-summary-blank { - padding:10px; -} - -/* Capability matrix semantic coloring. */ -th.color-metadata, td.color-metadata { - background-color:#fff; - border-color:#fff; - color:#000; -} - -th.color-capability { - background-color:#333; - border-color:#222; -} - -th.color-platform { - background-color:#333; - border-color:#222; -} - -td.color-blank { - background-color:#fff; - color:#fff; -} - -/* Capability matrix semantic formatting */ -th.format-category { - vertical-align:text-top; - font-size:20px; - text-align:center; -} - -th.format-capability { - text-align:right; - white-space:nowrap; -} - -th.format-platform { - text-align:center; -} - -/* Capability matrix expand/collapse details toggle. */ -div.cap-toggle { - border-color:#000; - color:#000; - padding-top:1.5ex; - border-style:solid; - border-width:0px; - text-align:center; - cursor:pointer; - position:absolute; - font-size:16px; - font-weight:normal; +#table-link { + margin-bottom: 84px; + font-size: 14px; + font-weight: 700; + line-height: 1.14; + letter-spacing: 0.6px; + color: #f26628; + text-transform: uppercase; + &:hover { + text-decoration: none; + color: #f2804d; + } } diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index 81181e2cab8c..dfddc141600c 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -46,5 +46,6 @@ @import "_quotes.scss"; @import "_blog.scss"; @import "_lists.scss"; +@import "_traits.scss"; @import "_navbar-desktop.scss"; @import "_search.scss"; diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index 333039f39a4f..a3a6c972d5bf 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -1,12 +1,15 @@ --- title: "Beam Contribution Guide" +type: "contribute" +layout: "arrow_template" aliases: - - /contribution-guide/ - - /contribute/contribution-guide/ - - /docs/contribute/ - - /contribute/source-repository/ - - /contribute/design-principles/ + - /contribution-guide/ + - /contribute/contribution-guide/ + - /docs/contribute/ + - /contribute/source-repository/ + - /contribute/design-principles/ --- + -# Apache Beam Contribution Guide +# Contribution guide + + -If you have questions, please [reach out to the Beam community](/contribute/get-help). +
-There are lots of opportunities to contribute: +There are lots of opportunities to contribute. You can for example: - - ask or answer questions on [user@beam.apache.org](/community/contact-us/) or +- ask or answer questions on [user@beam.apache.org](/community/contact-us/) or [stackoverflow](https://stackoverflow.com/questions/tagged/apache-beam) - review proposed design ideas on [dev@beam.apache.org](/community/contact-us/) - improve the documentation @@ -49,17 +57,22 @@ There are lots of opportunities to contribute: https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) - organize local meetups of users or contributors to Apache Beam -Most importantly, if you have an idea of how to contribute, then do it! +
+ + + +
+ Below is a tutorial for contributing code to Beam, covering our tools and typical process in detail. ### Prerequisites -To contribute code, you need - - a GitHub account - a Linux, macOS, or Microsoft Windows development environment with Java JDK 8 installed - [Docker](https://www.docker.com/) installed for some tasks including building worker containers and testing this website @@ -71,6 +84,8 @@ To contribute code, you need Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache Software Foundation (ASF). + + To install these in a Debian-based distribution: ``` @@ -127,8 +142,7 @@ script which is part of the Beam repo: 1. Assign the issue to yourself. To get the permission to do so, email the [dev@ mailing list](/community/contact-us) to introduce yourself and to be added as a contributor in the Beam issue tracker including your - ASF Jira Username. For example [this welcome email]( - https://lists.apache.org/thread.html/e6018c2aaf7dc7895091434295e5b0fafe192b975e3e3761fcf0cda7@%3Cdev.beam.apache.org%3E). + ASF Jira Username. For example [this welcome email](https://lists.apache.org/thread.html/e6018c2aaf7dc7895091434295e5b0fafe192b975e3e3761fcf0cda7@%3Cdev.beam.apache.org%3E). 1. If your change is large or it is your first change, it is a good idea to [discuss it on the dev@ mailing list](/community/contact-us/). 1. For large changes create a design doc @@ -233,6 +247,8 @@ script which is part of the Beam repo: all commits in the PR during merge, however if a PR has a mixture of independent changes that should not be squashed, and fixup commits, then the PR author should help squashing fixup commits to maintain a clean commmit history. +
+ ## When will my change show up in an Apache Beam release? Apache Beam makes minor releases every 6 weeks. Apache Beam has a diff --git a/website/www/site/content/en/contribute/attributes.md b/website/www/site/content/en/contribute/attributes.md new file mode 100644 index 000000000000..27598eb58a9b --- /dev/null +++ b/website/www/site/content/en/contribute/attributes.md @@ -0,0 +1,66 @@ +--- +title: "Attributes of a Beam community member" +layout: "arrow_template" +--- + + + +# Attributes of a Beam community member + +{{< figure src="/images/community/messages-icon.svg" >}} + +## Knows, upholds, and reinforces the Beam community’s practices + +- They have a proven commitment to the project +- They share their intentions with the community +- They accept and integrate community feedback in their plans, designs, code, etc. +- They earnestly try to make Beam better with their contributions + +In particular, if a code contributor: + +- They earnestly try to make Beam better with their own code +- They earnestly try to make Beam better with code review +- They accept and integrate feedback on their code +- They know, follow, and enforce Beam’s practices while reviewing/merging code - style, documentation, testing, backward compatibility, etc. + +{{< figure src="/images/community/beam-logo-icon.svg" >}} + +## Knows, upholds, and reinforces the Apache Software Foundation code of conduct + +In particular, we manifestly strive to: + +- Be open +- Be empathetic +- Be welcoming +- Be friendly +- Be patient +- Be collaborative +- Be inquisitive +- Be careful in the words that they choose + +[To learn more see the ASF documentation.](https://httpd.apache.org/docs/) + +{{< figure src="/images/community/diamond-icon.svg" >}} + +## Knows, upholds, and reinforces the responsibilities of an Apache Software Foundation committer + +- They help create a product that will outlive the interest of any particular volunteer (including themselves) +- They grow and maintain the health of the Apache community +- They help out with surrounding work, such as the website & documentation +- They help users +- They can be trusted to decide when code is ready for release, or when to ask someone else to make the judgment +- They can be trusted to decide when to merge code (if a code contributor) or when to ask someone else to make the judgment + +[To learn more see the ASF documentation.](https://httpd.apache.org/docs/) diff --git a/website/www/site/content/en/contribute/become-a-committer.md b/website/www/site/content/en/contribute/become-a-committer.md index 5d86beb471e0..822ee62a56c4 100644 --- a/website/www/site/content/en/contribute/become-a-committer.md +++ b/website/www/site/content/en/contribute/become-a-committer.md @@ -1,6 +1,7 @@ --- title: "Become A Committer" --- + -# Become a Committer - -An Apache Beam -[committer](https://www.apache.org/foundation/how-it-works.html#committers) has -write access to the repository for merging pull requests, but you don't have -to be a code contributor to become a committer. Becoming a committer means that -you have the project's trust. Read the [ASF -documentation](https://www.apache.org/dev/committers.html#committer-responsibilities) -for more about being a committer in the Apache Software Foundation. - -The [PMC](https://www.apache.org/foundation/how-it-works.html#pmc-members) -makes someone a committer via nomination, discussion, and then majority vote. -We use data from as many sources as possible to inform our reasoning. Here are -some examples: - - - [dev@ archives](https://lists.apache.org/list.html?dev@beam.apache.org) and [statistics](https://lists.apache.org/trends.html?dev@beam.apache.org) - - [user@ archives](https://lists.apache.org/list.html?user@beam.apache.org) and [statistics](https://lists.apache.org/trends.html?user@beam.apache.org) - - [`apache-beam` StackOverflow tag](https://stackoverflow.com/questions/tagged/apache-beam) - - Git metrics for [Beam](https://github.com/apache/beam/graphs/contributors) - - Code reviews given and received on - [Beam](https://github.com/apache/beam/pulls) - - Clear areas of ownership (a runner, a DSL, IO connector, documentation, - etc.) - - Public events - - Firsthand PMC testimonials - -The PMC has assembled the following set of guidelines for becoming a committer. - -## An Apache Beam committer... - -### Takes many forms - -There are many actions other than coding that build the trust we place in a -committer - code review, design discussion, user support, community outreach, improving -infrastructure, documentation, project management, etc. - -### Knows, upholds, and reinforces the Apache Software Foundation code of conduct +# Become a Beam Committer -See the [ASF -documentation](https://www.apache.org/foundation/policies/conduct.html). In -particular, they manifestly strive to: +An Apache Beam [committer](https://www.apache.org/foundation/how-it-works.html#committers) takes many forms. There are many actions other than coding that build the trust we place in a committer - code review, design discussion, user support, community outreach, improving infrastructure, documentation, project management, etc. - - Be open - - Be empathetic - - Be welcoming - - Be friendly - - Be patient - - Be collaborative - - Be inquisitive - - Be careful in the words that they choose +### What does it mean being a commiter? -### Knows, upholds, and reinforces the responsibilities of an Apache Software Foundation committer +An Apache Beam committer has write access to the repository for merging pull requests, but you don’t have to be a code contributor to become a committer. Becoming a committer means that you have the project’s trust. Read the [ASF documentation](https://www.apache.org/dev/committers.html#committer-responsibilities) for more about being a committer in the Apache Software Foundation. -See the [ASF documentation](https://www.apache.org/dev/committers.html#committer-responsibilities). +The [PMC](https://www.apache.org/foundation/how-it-works.html#pmc-members) makes someone a committer via nomination, discussion, and then majority vote. We use data from as many sources as possible to inform our reasoning. - - They help create a product that will outlive the interest of any particular - volunteer (including themselves) - - They grow and maintain the health of the Apache community - - They help out with surrounding work, such as the website & documentation - - They help users - - They can be trusted to decide when code is ready for release, or when to ask - someone else to make the judgment - - They can be trusted to decide when to merge code (if a code contributor) or - when to ask someone else to make the judgment +### Different Ways you can contribute +Everyone is welcome to join and contribute to the project in many ways, not only with code contributions. Things like asking questions, reporting bugs, proposing new features, improving documentation or the website, organizing events or writing blog posts, are also welcome and recognized. -### Knows, upholds, and reinforces the Beam community’s practices +{{< contributor/list_with_icons ways_of_contribution >}} - - They have a proven commitment to the project - - They share their intentions with the community - - They accept and integrate community feedback in their plans, designs, - code, etc. - - They earnestly try to make Beam better with their contributions - - In particular, if a code contributor: - - They earnestly try to make Beam better with their own code - - They earnestly try to make Beam better with code review - - They accept and integrate feedback on their code - - They know, follow, and enforce Beam’s practices while - reviewing/merging code - style, documentation, testing, backward - compatibility, etc. +### What are the traits of an Apache Beam Committer? +{{< contributor/row_of_traits committer_traits >}} diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/cogroupbykey.md b/website/www/site/content/en/documentation/transforms/python/aggregation/cogroupbykey.md index f80c3b20affe..d5dac4ed9d73 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/cogroupbykey.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/cogroupbykey.md @@ -38,7 +38,7 @@ Then, we apply `CoGroupByKey` to join both `PCollection`s using their keys. `CoGroupByKey` expects a dictionary of named keyed `PCollection`s, and produces elements joined by their keys. The values of each output element are dictionaries where the names correspond to the input dictionary, with lists of all the values found for that key. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/cogroupbykey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/cogroupbykey.py" cogroupbykey >}} {{< /highlight >}} @@ -50,9 +50,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/cogroupbykey_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/cogroupbykey.py" >}} - ## Related transforms * [CombineGlobally](/documentation/transforms/python/aggregation/combineglobally) to combine elements. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/combineglobally.md b/website/www/site/content/en/documentation/transforms/python/aggregation/combineglobally.md index 53e2fc0bdb3f..ab25dca7b683 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/combineglobally.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/combineglobally.md @@ -36,7 +36,7 @@ Then, we apply `CombineGlobally` in multiple ways to combine all the elements in We define a function `get_common_items` which takes an `iterable` of sets as an input, and calculates the intersection (common items) of those sets. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_function >}} {{< /highlight >}} @@ -48,14 +48,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" common_items >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ### Example 2: Combining with a lambda function We can also use lambda functions to simplify **Example 1**. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_lambda >}} {{< /highlight >}} @@ -67,9 +64,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" common_items >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ### Example 3: Combining with multiple arguments You can pass functions with multiple arguments to `CombineGlobally`. @@ -77,7 +71,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, the lambda function takes `sets` and `exclude` as arguments. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_multiple_arguments >}} {{< /highlight >}} @@ -89,9 +83,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" common_items_with_exceptions >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ### Example 4: Combining with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -100,7 +91,7 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `'🥕'` as a singleton. We then use that value to exclude specific items. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_side_inputs_singleton >}} {{< /highlight >}} @@ -112,16 +103,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" common_items_with_exceptions >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ### Example 5: Combining with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_side_inputs_iter >}} {{< /highlight >}} @@ -133,9 +121,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" common_items_with_exceptions >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -146,7 +131,7 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_side_inputs_dict >}} {{< /highlight >}} @@ -158,9 +143,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" custom_common_items >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ### Example 7: Combining with a `CombineFn` The more general way to combine elements, and the most flexible, is with a class that inherits from `CombineFn`. @@ -179,7 +161,7 @@ The more general way to combine elements, and the most flexible, is with a class * [`CombineFn.extract_output()`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.CombineFn.extract_output): It allows to do additional calculations before extracting a result. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" combineglobally_combinefn >}} {{< /highlight >}} @@ -191,9 +173,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" percentages >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineglobally.py" >}} - ## Related transforms You can use the following combiner transforms: diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/combineperkey.md b/website/www/site/content/en/documentation/transforms/python/aggregation/combineperkey.md index cc6ffd1e1273..196fcd62450d 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/combineperkey.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/combineperkey.md @@ -38,7 +38,7 @@ We use the function [`sum`](https://docs.python.org/3/library/functions.html#sum) which takes an `iterable` of numbers and adds them together. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_simple >}} {{< /highlight >}} @@ -50,14 +50,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 2: Combining with a function We define a function `saturated_sum` which takes an `iterable` of numbers and adds them together, up to a predefined maximum number. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_function >}} {{< /highlight >}} @@ -69,14 +66,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 3: Combining with a lambda function We can also use lambda functions to simplify **Example 2**. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_lambda >}} {{< /highlight >}} @@ -88,9 +82,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 4: Combining with multiple arguments You can pass functions with multiple arguments to `CombinePerKey`. @@ -98,7 +89,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, the lambda function takes `values` and `max_value` as arguments. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_multiple_arguments >}} {{< /highlight >}} @@ -110,9 +101,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 5: Combining with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -121,7 +109,7 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `8` as a singleton. We then use that value as the `max_value` for our saturated sum. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_side_inputs_singleton >}} {{< /highlight >}} @@ -133,16 +121,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 6: Combining with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_side_inputs_iter >}} {{< /highlight >}} @@ -154,9 +139,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" bounded_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -167,7 +149,7 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_side_inputs_dict >}} {{< /highlight >}} @@ -179,9 +161,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" bounded_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ### Example 8: Combining with a `CombineFn` The more general way to combine elements, and the most flexible, is with a class that inherits from `CombineFn`. @@ -200,7 +179,7 @@ The more general way to combine elements, and the most flexible, is with a class * [`CombineFn.extract_output()`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.CombineFn.extract_output): It allows to do additional calculations before extracting a result. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" combineperkey_combinefn >}} {{< /highlight >}} @@ -212,9 +191,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" average >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combineperkey.py" >}} - ## Related transforms You can use the following combiner transforms: diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/combinevalues.md b/website/www/site/content/en/documentation/transforms/python/aggregation/combinevalues.md index c231e116ff9b..24d88e4f08e0 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/combinevalues.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/combinevalues.md @@ -39,7 +39,7 @@ We use the function [`sum`](https://docs.python.org/3/library/functions.html#sum) which takes an `iterable` of numbers and adds them together. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_simple >}} {{< /highlight >}} @@ -51,9 +51,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 2: Combining with a function We want the sum to be bounded up to a maximum value, so we use @@ -61,7 +58,7 @@ We want the sum to be bounded up to a maximum value, so we use We define a function `saturated_sum` which takes an `iterable` of numbers and adds them together, up to a predefined maximum number. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_function >}} {{< /highlight >}} @@ -73,14 +70,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 3: Combining with a lambda function We can also use lambda functions to simplify **Example 2**. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_lambda >}} {{< /highlight >}} @@ -92,9 +86,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 4: Combining with multiple arguments You can pass functions with multiple arguments to `CombineValues`. @@ -102,7 +93,7 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, the lambda function takes `values` and `max_value` as arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_multiple_arguments >}} {{< /highlight >}} @@ -114,9 +105,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 5: Combining with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -125,7 +113,7 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `8` as a singleton. We then use that value as the `max_value` for our saturated sum. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_side_inputs_singleton >}} {{< /highlight >}} @@ -137,16 +125,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" saturated_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 6: Combining with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_side_inputs_iter >}} {{< /highlight >}} @@ -158,9 +143,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" bounded_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -171,7 +153,7 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_side_inputs_dict >}} {{< /highlight >}} @@ -183,9 +165,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" bounded_total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ### Example 8: Combining with a `CombineFn` The more general way to combine elements, and the most flexible, is with a class that inherits from `CombineFn`. @@ -204,7 +183,7 @@ The more general way to combine elements, and the most flexible, is with a class * [`CombineFn.extract_output()`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.CombineFn.extract_output): It allows to do additional calculations before extracting a result. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" combinevalues_combinefn >}} {{< /highlight >}} @@ -216,9 +195,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" percentages_per_season >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/combinevalues.py" >}} - ## Related transforms You can use the following combiner transforms: @@ -230,4 +206,4 @@ You can use the following combiner transforms: * [Top](/documentation/transforms/python/aggregation/top) * [Sample](/documentation/transforms/python/aggregation/sample) -{{< button-pydoc path="apache_beam.transforms.core" class="CombineValues" >}} \ No newline at end of file +{{< button-pydoc path="apache_beam.transforms.core" class="CombineValues" >}} diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/count.md b/website/www/site/content/en/documentation/transforms/python/aggregation/count.md index f4d332ed2fc9..5aa6e1b8252f 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/count.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/count.md @@ -32,7 +32,7 @@ Then, we apply `Count` to get the total number of elements in different ways. We use `Count.Globally()` to count *all* elements in a `PCollection`, even if there are duplicate elements. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" count_globally >}} {{< /highlight >}} @@ -44,14 +44,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count_test.py" total_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} - ### Example 2: Counting elements for each key We use `Count.PerKey()` to count the elements for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" count_per_key >}} {{< /highlight >}} @@ -63,14 +60,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count_test.py" total_elements_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} - ### Example 3: Counting all unique elements We use `Count.PerElement()` to count the only the unique elements in a `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" count_per_element >}} {{< /highlight >}} @@ -82,9 +76,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/count_test.py" total_unique_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/count.py" >}} - ## Related transforms N/A diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/distinct.md b/website/www/site/content/en/documentation/transforms/python/aggregation/distinct.md index e0abca48ee7a..08db6bd987c5 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/distinct.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/distinct.md @@ -29,7 +29,7 @@ In the following example, we create a pipeline with two `PCollection`s of produc We use `Distinct` to get rid of duplicate elements, which outputs a `PCollection` of all the unique elements. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/distinct.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/distinct.py" distinct >}} {{< /highlight >}} @@ -41,9 +41,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/distinct_test.py" unique_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/distinct.py" >}} - ## Related transforms * [Count](/documentation/transforms/python/aggregation/count) counts the number of elements within each aggregation. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/groupby.md b/website/www/site/content/en/documentation/transforms/python/aggregation/groupby.md index ad6c17b8ec3f..df281bdabe0a 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/groupby.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/groupby.md @@ -32,7 +32,7 @@ In the following example, we create a pipeline with a `PCollection` of fruits. We use `GroupBy` to group all fruits by the first letter of their name. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_expr >}} {{< /highlight >}} @@ -44,13 +44,9 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_expr_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} - - We can group by a composite key consisting of multiple properties if desired. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_two_exprs >}} {{< /highlight >}} @@ -65,10 +61,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_two_exprs_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} - - In the case that the property one wishes to group by is an attribute, a string may be passed to `GroupBy` in the place of a callable expression. For example, suppose I have the following data @@ -79,7 +71,7 @@ suppose I have the following data We can then do -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_attr >}} {{< /highlight >}} @@ -91,12 +83,9 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_attr_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} - It is possible to mix and match attributes and expressions, for example -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_attr_expr >}} {{< /highlight >}} @@ -108,9 +97,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" groupby_attr_expr_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}}. - ## Aggregation Grouping is often used in conjunction with aggregation, and the @@ -122,7 +108,7 @@ by, and finally a field name in which to store the result. For example, suppose one wanted to compute the amount of each fruit to buy. One could write -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" simple_aggregate >}} {{< /highlight >}} @@ -134,13 +120,10 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" simple_aggregate_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}}. - Similar to the parameters in `GroupBy`, one can also aggregate multiple fields and by expressions. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" expr_aggregate >}} {{< /highlight >}} @@ -152,13 +135,10 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" expr_aggregate_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}}. - One can, of course, aggregate the same field multiple times as well. This example also illustrates a global grouping, as the grouping key is empty. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" global_aggregate >}} {{< /highlight >}} @@ -170,10 +150,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" global_aggregate_result >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py" >}}. - - ## Related transforms * [CombinePerKey](/documentation/transforms/python/aggregation/combineperkey) for combining with a single CombineFn. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/groupbykey.md b/website/www/site/content/en/documentation/transforms/python/aggregation/groupbykey.md index f313e9d0dbf4..06ff183f833b 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/groupbykey.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/groupbykey.md @@ -32,7 +32,7 @@ In the following example, we create a pipeline with a `PCollection` of produce k We use `GroupByKey` to group all the produce for each season. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupbykey.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupbykey.py" groupbykey >}} {{< /highlight >}} @@ -44,9 +44,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupbykey_test.py" produce_counts >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupbykey.py" >}} - ## Related transforms * [GroupBy](/documentation/transforms/python/aggregation/groupby) for grouping by arbitrary properties of the elements. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/groupintobatches.md b/website/www/site/content/en/documentation/transforms/python/aggregation/groupintobatches.md index d9917fd5ae4b..a7c9cd9b2348 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/groupintobatches.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/groupintobatches.md @@ -29,7 +29,7 @@ In the following example, we create a pipeline with a `PCollection` of produce b We use `GroupIntoBatches` to get fixed-sized batches for every key, which outputs a list of elements for every key. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupintobatches.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupintobatches.py" groupintobatches >}} {{< /highlight >}} @@ -41,9 +41,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupintobatches_test.py" batches_with_keys >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupintobatches.py" >}} - ## Related transforms diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/latest.md b/website/www/site/content/en/documentation/transforms/python/aggregation/latest.md index f9692b10f73a..613858c44752 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/latest.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/latest.md @@ -33,7 +33,7 @@ We use `Latest` to get the element with the latest timestamp from the `PCollecti We use `Latest.Globally()` to get the element with the latest timestamp in the entire `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" latest_globally >}} {{< /highlight >}} @@ -45,14 +45,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest_test.py" latest_element >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" >}} - ### Example 2: Latest elements for each key We use `Latest.PerKey()` to get the elements with the latest timestamp for each key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" latest_per_key >}} {{< /highlight >}} @@ -64,9 +61,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest_test.py" latest_elements_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/latest.py" >}} - ## Related transforms * [Sample](/documentation/transforms/python/aggregation/sample) randomly takes some number of elements in a collection. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/max.md b/website/www/site/content/en/documentation/transforms/python/aggregation/max.md index 7769b814517c..37210888226c 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/max.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/max.md @@ -32,7 +32,7 @@ Then, we get the element with the maximum value in different ways. We use `Combine.Globally()` to get the maximum element from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" max_globally >}} {{< /highlight >}} @@ -44,14 +44,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/max_test.py" max_element >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" >}} - ### Example 2: Maximum elements for each key We use `Combine.PerKey()` to get the maximum element for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" max_per_key >}} {{< /highlight >}} @@ -63,9 +60,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/max_test.py" elements_with_max_value_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/max.py" >}} - ## Related transforms * [CombineGlobally](/documentation/transforms/python/aggregation/combineglobally) diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/mean.md b/website/www/site/content/en/documentation/transforms/python/aggregation/mean.md index 69fa5ade447d..76a2d0082ed1 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/mean.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/mean.md @@ -34,7 +34,7 @@ Then, we get the element with the average value in different ways. We use `Mean.Globally()` to get the average of the elements from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" mean_globally >}} {{< /highlight >}} @@ -46,14 +46,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean_test.py" mean_element >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" >}} - ### Example 2: Mean of elements for each key We use `Mean.PerKey()` to get the average of the elements for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" mean_per_key >}} {{< /highlight >}} @@ -65,9 +62,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean_test.py" elements_with_mean_value_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/mean.py" >}} - ## Related transforms * [CombineGlobally](/documentation/transforms/python/aggregation/combineglobally) diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/min.md b/website/www/site/content/en/documentation/transforms/python/aggregation/min.md index 8f385d20af69..62f9d15d4904 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/min.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/min.md @@ -32,7 +32,7 @@ Then, we get the element with the minimum value in different ways. We use `Combine.Globally()` to get the minimum element from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" min_globally >}} {{< /highlight >}} @@ -44,14 +44,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/min_test.py" min_element >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" >}} - ### Example 2: Minimum elements for each key We use `Combine.PerKey()` to get the minimum element for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" min_per_key >}} {{< /highlight >}} @@ -63,9 +60,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/min_test.py" elements_with_min_value_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/min.py" >}} - ## Related transforms * [CombineGlobally](/documentation/transforms/python/aggregation/combineglobally) diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/sample.md b/website/www/site/content/en/documentation/transforms/python/aggregation/sample.md index 33fc8f60235b..bd822ea89d4e 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/sample.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/sample.md @@ -34,7 +34,7 @@ Then, we get a random sample of elements in different ways. We use `Sample.FixedSizeGlobally()` to get a fixed-size random sample of elements from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" sample_fixed_size_globally >}} {{< /highlight >}} @@ -46,14 +46,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample_test.py" sample >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" >}} - ### Example 2: Sample elements for each key We use `Sample.FixedSizePerKey()` to get fixed-size random samples for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" sample_fixed_size_per_key >}} {{< /highlight >}} @@ -65,9 +62,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample_test.py" samples_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sample.py" >}} - ## Related transforms * [Top](/documentation/transforms/python/aggregation/top) finds the largest or smallest element. diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/sum.md b/website/www/site/content/en/documentation/transforms/python/aggregation/sum.md index a9aef38d4c21..9ceda5cbb084 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/sum.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/sum.md @@ -32,7 +32,7 @@ Then, we get the sum of all the element values in different ways. We use `Combine.Globally()` to get sum of all the element values from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" sum_globally >}} {{< /highlight >}} @@ -44,14 +44,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum_test.py" total >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" >}} - ### Example 2: Sum of the elements for each key We use `Combine.PerKey()` to get the sum of all the element values for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" sum_per_key >}} {{< /highlight >}} @@ -63,9 +60,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum_test.py" totals_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/sum.py" >}} - ## Related transforms * [CombineGlobally](/documentation/transforms/python/aggregation/combineglobally) diff --git a/website/www/site/content/en/documentation/transforms/python/aggregation/top.md b/website/www/site/content/en/documentation/transforms/python/aggregation/top.md index 419c0f81fb58..8c93499d049b 100644 --- a/website/www/site/content/en/documentation/transforms/python/aggregation/top.md +++ b/website/www/site/content/en/documentation/transforms/python/aggregation/top.md @@ -34,7 +34,7 @@ Then, we get the largest or smallest elements in different ways. We use `Top.Largest()` to get the largest elements from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_largest >}} {{< /highlight >}} @@ -46,14 +46,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" largest_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ### Example 2: Largest elements for each key We use `Top.LargestPerKey()` to get the largest elements for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_largest_per_key >}} {{< /highlight >}} @@ -65,14 +62,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" largest_elements_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ### Example 3: Smallest elements from a PCollection We use `Top.Smallest()` to get the smallest elements from the *entire* `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_smallest >}} {{< /highlight >}} @@ -84,14 +78,11 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" smallest_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ### Example 4: Smallest elements for each key We use `Top.SmallestPerKey()` to get the smallest elements for each unique key in a `PCollection` of key-values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_smallest_per_key >}} {{< /highlight >}} @@ -103,9 +94,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" smallest_elements_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ### Example 5: Custom elements from a PCollection We use `Top.Of()` to get elements with customized rules from the *entire* `PCollection`. @@ -113,7 +101,7 @@ We use `Top.Of()` to get elements with customized rules from the *entire* `PColl You can change how the elements are compared with `key`. By default you get the largest elements, but you can get the smallest by setting `reverse=True`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_of >}} {{< /highlight >}} @@ -125,9 +113,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" shortest_elements >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ### Example 6: Custom elements for each key We use `Top.PerKey()` to get elements with customized rules for each unique key in a `PCollection` of key-values. @@ -135,7 +120,7 @@ We use `Top.PerKey()` to get elements with customized rules for each unique key You can change how the elements are compared with `key`. By default you get the largest elements, but you can get the smallest by setting `reverse=True`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" top_per_key >}} {{< /highlight >}} @@ -147,9 +132,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/aggregation/top_test.py" shortest_elements_per_key >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/aggregation/top.py" >}} - ## Related transforms * [Sample](/documentation/transforms/python/aggregation/sample) to combine elements. Takes samples of the elements in a collection. diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md b/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md index 3f602441932b..3b1b46e35239 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/filter.md @@ -36,7 +36,7 @@ Then, we apply `Filter` in multiple ways to filter out produce by their duration We define a function `is_perennial` which returns `True` if the element's duration equals `'perennial'`, and `False` otherwise. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_function >}} {{< /highlight >}} @@ -48,15 +48,12 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - ### Example 2: Filtering with a lambda function We can also use lambda functions to simplify **Example 1**. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_lambda >}} {{< /highlight >}} @@ -68,10 +65,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - ### Example 3: Filtering with multiple arguments You can pass functions with multiple arguments to `Filter`. @@ -79,7 +72,8 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `has_duration` takes `plant` and `duration` as arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_multiple_arguments >}} {{< /highlight >}} @@ -91,10 +85,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - ### Example 4: Filtering with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -103,7 +93,8 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `'perennial'` as a singleton. We then use that value to filter out perennials. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_singleton >}} {{< /highlight >}} @@ -115,17 +106,14 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - ### Example 5: Filtering with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_iter >}} {{< /highlight >}} @@ -137,10 +125,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" valid_plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -151,7 +135,8 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" filter_side_inputs_dict >}} {{< /highlight >}} @@ -163,10 +148,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter_test.py" perennials >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/filter.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/filter" >}} - ## Related transforms * [FlatMap](/documentation/transforms/python/elementwise/flatmap) behaves the same as `Map`, but for diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md b/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md index 7c7baa117b76..15648b986e2f 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/flatmap.md @@ -37,7 +37,8 @@ where each of the output `iterable`'s elements is an element of the resulting `P We use the function `str.split` which takes a single `str` element and outputs a `list` of `str`s. This pipeline splits the input element using whitespaces, creating a list of zero or more elements. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_simple >}} {{< /highlight >}} @@ -49,15 +50,12 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 2: FlatMap with a function We define a function `split_words` which splits an input `str` element using the delimiter `','` and outputs a `list` of `str`s. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_function >}} {{< /highlight >}} @@ -69,17 +67,14 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 3: FlatMap with a lambda function For this example, we want to flatten a `PCollection` of lists of `str`s into a `PCollection` of `str`s. Each input element is already an `iterable`, where each element is what we want in the resulting `PCollection`. We use a lambda function that returns the same input element it received. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_lambda >}} {{< /highlight >}} @@ -91,17 +86,14 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 4: FlatMap with a generator For this example, we want to flatten a `PCollection` of lists of `str`s into a `PCollection` of `str`s. We use a generator to iterate over the input list and yield each of the elements. Each yielded result in the generator is an element in the resulting `PCollection`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_generator >}} {{< /highlight >}} @@ -113,16 +105,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 5: FlatMapTuple for key-value pairs If your `PCollection` consists of `(key, value)` pairs, you can use `FlatMapTuple` to unpack them into different function arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_tuple >}} {{< /highlight >}} @@ -134,10 +123,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 6: FlatMap with multiple arguments You can pass functions with multiple arguments to `FlatMap`. @@ -145,7 +130,8 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `split_words` takes `text` and `delimiter` as arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_multiple_arguments >}} {{< /highlight >}} @@ -157,10 +143,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 7: FlatMap with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -169,7 +151,8 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `','` as a singleton. We then use that value as the delimiter for the `str.split` method. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_singleton >}} {{< /highlight >}} @@ -181,17 +164,14 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ### Example 8: FlatMap with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_iter >}} {{< /highlight >}} @@ -203,10 +183,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -217,7 +193,8 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" flatmap_side_inputs_dict >}} {{< /highlight >}} @@ -229,10 +206,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" valid_plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/flatmap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/flatmap" >}} - ## Related transforms * [Filter](/documentation/transforms/python/elementwise/filter) is useful if the function is just diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md b/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md index 1bfac3bb94ce..d30717d3e814 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/keys.md @@ -28,7 +28,8 @@ Takes a collection of key-value pairs and returns the key of each element. In the following example, we create a pipeline with a `PCollection` of key-value pairs. Then, we apply `Keys` to extract the keys and discard the values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/keys" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys.py" keys >}} {{< /highlight >}} @@ -40,10 +41,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys_test.py" icons >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/keys.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/keys" >}} - ## Related transforms * [KvSwap](/documentation/transforms/python/elementwise/kvswap) swaps the key and value of each element. diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md b/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md index 259fe8902587..3c3a25518c82 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/kvswap.md @@ -29,7 +29,8 @@ which has each key and value swapped. In the following example, we create a pipeline with a `PCollection` of key-value pairs. Then, we apply `KvSwap` to swap the keys and values. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/kvswap" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap.py" kvswap >}} {{< /highlight >}} @@ -41,10 +42,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/kvswap.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/kvswap" >}} - ## Related transforms * [Keys](/documentation/transforms/python/elementwise/keys) for extracting the key of each component. diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/map.md b/website/www/site/content/en/documentation/transforms/python/elementwise/map.md index a6f3bdab9f52..d31bbabea2e2 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/map.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/map.md @@ -35,7 +35,8 @@ Then, we apply `Map` in multiple ways to transform every element in the `PCollec We use the function `str.strip` which takes a single `str` element and outputs a `str`. It strips the input element's whitespaces, including newlines and tabs. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_simple >}} {{< /highlight >}} @@ -47,15 +48,12 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 2: Map with a function We define a function `strip_header_and_newline` which strips any `'#'`, `' '`, and `'\n'` characters from each element. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_function >}} {{< /highlight >}} @@ -67,15 +65,12 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 3: Map with a lambda function We can also use lambda functions to simplify **Example 2**. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_lambda >}} {{< /highlight >}} @@ -87,10 +82,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 4: Map with multiple arguments You can pass functions with multiple arguments to `Map`. @@ -98,7 +89,8 @@ They are passed as additional positional arguments or keyword arguments to the f In this example, `strip` takes `text` and `chars` as arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_multiple_arguments >}} {{< /highlight >}} @@ -110,16 +102,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 5: MapTuple for key-value pairs If your `PCollection` consists of `(key, value)` pairs, you can use `MapTuple` to unpack them into different function arguments. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_tuple >}} {{< /highlight >}} @@ -131,10 +120,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 6: Map with side inputs as singletons If the `PCollection` has a single value, such as the average from another computation, @@ -143,7 +128,8 @@ passing the `PCollection` as a *singleton* accesses that value. In this example, we pass a `PCollection` the value `'# \n'` as a singleton. We then use that value as the characters for the `str.strip` method. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_singleton >}} {{< /highlight >}} @@ -155,17 +141,14 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ### Example 7: Map with side inputs as iterators If the `PCollection` has multiple values, pass the `PCollection` as an *iterator*. This accesses elements lazily as they are needed, so it is possible to iterate over large `PCollection`s that won't fit into memory. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_iter >}} {{< /highlight >}} @@ -177,10 +160,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - > **Note**: You can pass the `PCollection` as a *list* with `beam.pvalue.AsList(pcollection)`, > but this requires that all the elements fit into memory. @@ -191,7 +170,8 @@ Each element must be a `(key, value)` pair. Note that all the elements of the `PCollection` must fit into memory for this. If the `PCollection` won't fit into memory, use `beam.pvalue.AsIter(pcollection)` instead. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" map_side_inputs_dict >}} {{< /highlight >}} @@ -203,10 +183,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/map_test.py" plant_details >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/map.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/map" >}} - ## Related transforms * [FlatMap](/documentation/transforms/python/elementwise/flatmap) behaves the same as `Map`, but for diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md b/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md index 16089701b5fd..5021e577a4c4 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/pardo.md @@ -41,7 +41,8 @@ which stores the `delimiter` as an object field. The `process` method is called once per element, and it can yield zero or more output elements. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn >}} {{}} @@ -53,10 +54,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} - ### Example 2: ParDo with timestamp and window information In this example, we add new parameters to the `process` method to bind parameter values at runtime. @@ -70,7 +67,8 @@ In this example, we add new parameters to the `process` method to bind parameter [`apache_beam.transforms.window.*Window`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.window.html) object. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_params >}} {{}} @@ -82,10 +80,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" dofn_params >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} - ### Example 3: ParDo with DoFn methods A [`DoFn`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.DoFn) @@ -131,7 +125,8 @@ starts and finishes with `start_bundle` and `finish_bundle`. Note that `teardown` is called as a *best effort* and is *not guaranteed*. For example, if the worker crashes, `teardown` might not be called. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" pardo_dofn_methods >}} {{}} @@ -143,10 +138,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" results >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/pardo.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/pardo" >}} - > *Known issues:* > > * [[BEAM-7885]](https://issues.apache.org/jira/browse/BEAM-7885) diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md b/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md index 05897053455e..fbfe8a99ac52 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/partition.md @@ -46,7 +46,8 @@ and it must return an integer in the range `0` to `num_partitions-1`. In the following example, we have a known list of durations. We partition the `PCollection` into one `PCollection` for every duration type. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_function >}} {{< /highlight >}} @@ -58,15 +59,12 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} - ### Example 2: Partition with a lambda function We can also use lambda functions to simplify **Example 1**. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_lambda >}} {{< /highlight >}} @@ -78,10 +76,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" partitions >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} - ### Example 3: Partition with multiple arguments You can pass functions with multiple arguments to `Partition`. @@ -114,7 +108,8 @@ identify the partition index to which that bucket corresponds. This `split_dataset` function is generic enough to support any number of partitions by any ratio. You might want to adapt the bucket assignment to use a more appropriate or randomized hash for your dataset. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" partition_multiple_arguments >}} {{< /highlight >}} @@ -126,10 +121,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition_test.py" train_test >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/partition.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/partition" >}} - ## Related transforms * [Filter](/documentation/transforms/python/elementwise/filter) is useful if the function is just diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md b/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md index c31fa452e571..6947fd18252a 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/regex.md @@ -61,7 +61,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres To start matching at any point instead of the beginning of the string, use [`Regex.find(regex)`](#example-4-regex-find). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches >}} {{< /highlight >}} @@ -73,10 +74,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 2: Regex match with all groups `Regex.all_matches` keeps only the elements that match the regular expression, @@ -90,7 +87,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres To start matching at any point instead of the beginning of the string, use [`Regex.find_all(regex, group=Regex.ALL, outputEmpty=False)`](#example-5-regex-find-all). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_all_matches >}} {{< /highlight >}} @@ -102,10 +100,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_all_matches >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 3: Regex match into key-value pairs `Regex.matches_kv` keeps only the elements that match the regular expression, @@ -120,7 +114,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres To start matching at any point instead of the beginning of the string, use [`Regex.find_kv(regex, keyGroup)`](#example-6-regex-find-as-key-value-pairs). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_matches_kv >}} {{< /highlight >}} @@ -132,10 +127,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches_kv >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 4: Regex find `Regex.find` keeps only the elements that match the regular expression, @@ -150,7 +141,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres If you need to match from the start only, consider using [`Regex.matches(regex)`](#example-1-regex-match). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find >}} {{< /highlight >}} @@ -162,10 +154,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_matches >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 5: Regex find all `Regex.find_all` returns a list of all the matches of the regular expression, @@ -180,7 +168,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres If you need to match all groups from the start only, consider using [`Regex.all_matches(regex)`](#example-2-regex-match-with-all-groups). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_all >}} {{< /highlight >}} @@ -192,10 +181,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_all >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 6: Regex find as key-value pairs `Regex.find_kv` returns a list of all the matches of the regular expression, @@ -211,7 +196,8 @@ To match until the end of the string, add `'$'` at the end of the regular expres If you need to match as key-value pairs from the start only, consider using [`Regex.matches_kv(regex)`](#example-3-regex-match-into-key-value-pairs). -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_find_kv >}} {{< /highlight >}} @@ -223,10 +209,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_find_kv >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 7: Regex replace all `Regex.replace_all` returns the string with all the occurrences of the regular expression replaced by another string. @@ -234,7 +216,8 @@ You can also use [backreferences](https://docs.python.org/3/library/re.html?highlight=backreference#re.sub) on the `replacement`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_all >}} {{< /highlight >}} @@ -246,10 +229,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_all >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 8: Regex replace first `Regex.replace_first` returns the string with the first occurrence of the regular expression replaced by another string. @@ -257,7 +236,8 @@ You can also use [backreferences](https://docs.python.org/3/library/re.html?highlight=backreference#re.sub) on the `replacement`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_replace_first >}} {{< /highlight >}} @@ -269,16 +249,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_replace_first >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ### Example 9: Regex split `Regex.split` returns the list of strings that were delimited by the specified regular expression. The argument `outputEmpty` is set to `False` by default, but can be set to `True` to keep empty items in the output list. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" regex_split >}} {{< /highlight >}} @@ -290,10 +267,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex_test.py" plants_split >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/regex.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/regex" >}} - ## Related transforms * [FlatMap](/documentation/transforms/python/elementwise/flatmap) behaves the same as `Map`, but for diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md b/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md index 2409524dcd2d..4b3d79acada0 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/tostring.md @@ -35,7 +35,8 @@ expect their input elements to be strings. The following example converts a `(key, value)` pair into a string delimited by `','`. You can specify a different delimiter using the `delimiter` argument. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_kvs >}} {{< /highlight >}} @@ -47,16 +48,13 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} - ### Example 2: Elements to string The following example converts a dictionary into a string. The string output will be equivalent to `str(element)`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_element >}} {{< /highlight >}} @@ -68,10 +66,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plant_lists >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} - ### Example 3: Iterables to string The following example converts an iterable, in this case a list of strings, @@ -79,7 +73,8 @@ into a string delimited by `','`. You can specify a different delimiter using the `delimiter` argument. The string output will be equivalent to `iterable.join(delimiter)`. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" tostring_iterables >}} {{< /highlight >}} @@ -91,10 +86,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" plants_csv >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/tostring.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/tostring" >}} - ## Related transforms * [Map](/documentation/transforms/python/elementwise/map) applies a simple 1-to-1 mapping function over each element in the collection diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/values.md b/website/www/site/content/en/documentation/transforms/python/elementwise/values.md index 90f706ee8c7d..63e324c10eb4 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/values.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/values.md @@ -28,7 +28,8 @@ Takes a collection of key-value pairs, and returns the value of each element. In the following example, we create a pipeline with a `PCollection` of key-value pairs. Then, we apply `Values` to extract the values and discard the keys. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/values.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/values" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/values.py" values >}} {{< /highlight >}} @@ -40,10 +41,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/values_test.py" plants >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/values.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/values" >}} - ## Related transforms * [Keys](/documentation/transforms/python/elementwise/keys) for extracting the key of each component. diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md b/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md index 6fa4702ee636..51c9c64abc23 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/withtimestamps.md @@ -33,7 +33,8 @@ The elements themselves often already contain a timestamp field. [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in the form of seconds. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_event_time >}} {{< /highlight >}} @@ -45,10 +46,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_timestamps >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} - To convert from a [`time.struct_time`](https://docs.python.org/3/library/time.html#time.struct_time) to `unix_time` you can use @@ -75,7 +72,8 @@ If each element has a chronological number, these numbers can be used as a [logical clock](https://en.wikipedia.org/wiki/Logical_clock). These numbers have to be converted to a *"seconds"* equivalent, which can be especially important depending on your windowing and late data rules. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_logical_clock >}} {{< /highlight >}} @@ -87,10 +85,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_events >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} - ### Example 3: Timestamp by processing time If the elements do not have any time data available, you can also use the current processing time for each element. @@ -99,7 +93,8 @@ Workers might have time deltas, so using this method is not a reliable way to do By using processing time, there is no way of knowing if data is arriving late because the timestamp is attached when the element *enters* into the pipeline. -{{< highlight py >}} +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" + notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" withtimestamps_processing_time >}} {{< /highlight >}} @@ -111,10 +106,6 @@ Output: {{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" plant_processing_times >}} {{< /highlight >}} -{{< buttons-code-snippet - py="sdks/python/apache_beam/examples/snippets/transforms/elementwise/withtimestamps.py" - notebook="examples/notebooks/documentation/transforms/python/elementwise/withtimestamps" >}} - ## Related transforms * [Reify](/documentation/transforms/python/elementwise/reify) converts between explicit and implicit forms of Beam values. diff --git a/website/www/site/data/en/committer_traits.yaml b/website/www/site/data/en/committer_traits.yaml new file mode 100644 index 000000000000..e8f551d048de --- /dev/null +++ b/website/www/site/data/en/committer_traits.yaml @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- body: { text: To learn more see the ASF documentation., link: https://www.apache.org/foundation/policies/conduct.html } + bolded: Knows, upholds, and reinforces the Apache Software Foundation code of conduct. + icon: icons/contributor/become a committer/messages-icon.svg +- body: { text: To learn more see the ASF documentation., link: https://infra.apache.org/committers.html#committer-responsibilities } + bolded: Knows, upholds, and reinforces the responsibilities of an Apache Software Foundation committer. + icon: icons/contributor/become a committer/beam-logo-icon.svg +- body: { text: To learn more see our best practices!, link: https://beam.apache.org/contribute/attributes/ } + bolded: Knows, upholds, and reinforces the Beam community’s practices. + icon: icons/contributor/become a committer/diamond-icon.svg diff --git a/website/www/site/data/en/ways_of_contribution.yaml b/website/www/site/data/en/ways_of_contribution.yaml new file mode 100644 index 000000000000..ccf870682608 --- /dev/null +++ b/website/www/site/data/en/ways_of_contribution.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- title: Code contributions + body: "Ways you can contribute code to the project include:" + list: + - Building a new feature like a new IO connector or PTransform + - Fixing a Bug + - Sharing a code example or template with the community + - Testing a release candidate + icon: icons/contributor/become a committer/code-icon.svg +- title: Non-code contributions + body: "Non-code contributions are ways to add value to the projects outside code, here are some examples of things that you can contribute:" + list: + - Organize a meetup + - Give a talk about Beam at a conference + - Update a typo in the documentation + - Write a blog post with useful tips about Beam + icon: icons/contributor/become a committer/file-icon.svg diff --git a/website/www/site/layouts/contribute/arrow_template.html b/website/www/site/layouts/contribute/arrow_template.html new file mode 100644 index 000000000000..5cada00e4436 --- /dev/null +++ b/website/www/site/layouts/contribute/arrow_template.html @@ -0,0 +1,40 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} + + + + + {{ partial "head.html" . }} + + + {{ partial "header.html" . }} +
+
+ + +
+ + + +
+ {{ .Content }} +
+
+{{ partial "footer.html" . }} + + diff --git a/website/www/site/layouts/partials/head.html b/website/www/site/layouts/partials/head.html index 60b103fac646..4e3c6032d717 100644 --- a/website/www/site/layouts/partials/head.html +++ b/website/www/site/layouts/partials/head.html @@ -40,6 +40,8 @@ + + diff --git a/website/www/site/layouts/partials/header.html b/website/www/site/layouts/partials/header.html index aff72e8fdc3e..6aa4a54ffd63 100644 --- a/website/www/site/layouts/partials/header.html +++ b/website/www/site/layouts/partials/header.html @@ -122,7 +122,7 @@
{{ T "nav-get-started" }}
  • -
    • {{ .text.text | markdownify }} +
    +
    diff --git a/website/www/site/layouts/shortcodes/buttons-code-snippet.html b/website/www/site/layouts/shortcodes/buttons-code-snippet.html deleted file mode 100644 index 875cf1219e68..000000000000 --- a/website/www/site/layouts/shortcodes/buttons-code-snippet.html +++ /dev/null @@ -1,32 +0,0 @@ -{{/* - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. -*/}} - -{{ $colab_logo := "https://github.com/googlecolab/open_in_colab/raw/master/images/icon32.png" }} -{{ $github_logo := "https://www.tensorflow.org/images/GitHub-Mark-32px.png" }} - -{{ $ctx := . }} -{{ with (.Get "py") }} - {{ with ($ctx.Get "notebook") }} - - -
    - Run code now Run code now -
    - {{ end }} - - -
    - View source code View source code -
    -{{ end }} - -




    diff --git a/website/www/site/layouts/shortcodes/contributor/list_with_icons.html b/website/www/site/layouts/shortcodes/contributor/list_with_icons.html new file mode 100644 index 000000000000..f6c9735b7541 --- /dev/null +++ b/website/www/site/layouts/shortcodes/contributor/list_with_icons.html @@ -0,0 +1,18 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. + */}} + + {{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} +
    + {{ range $item := $data }} + {{ partial "pillars/pillars-item" (dict "logo" .icon "header" .title "text" .body "list" .list) }} + {{ end }} +
    diff --git a/website/www/site/layouts/shortcodes/contributor/row_of_traits.html b/website/www/site/layouts/shortcodes/contributor/row_of_traits.html new file mode 100644 index 000000000000..7cf112a17530 --- /dev/null +++ b/website/www/site/layouts/shortcodes/contributor/row_of_traits.html @@ -0,0 +1,18 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. + */}} + + {{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} +
    + {{ range $item := $data }} + {{ partial "traits/traits-item" (dict "logo" .icon "text" .body "bolded" .bolded) }} + {{ end }} +
    diff --git a/website/www/site/layouts/shortcodes/highlight.html b/website/www/site/layouts/shortcodes/highlight.html index 7e3533ef45fb..9f0bb2252703 100644 --- a/website/www/site/layouts/shortcodes/highlight.html +++ b/website/www/site/layouts/shortcodes/highlight.html @@ -12,20 +12,50 @@ {{ $content := (trim .Inner "\n\r") | htmlUnescape | safeHTML }} {{ $ctx := . }} -{{ with (.Get "class") }} -
    - {{ with ($ctx.Get "lang") }} +{{ $language := .Get 0 }} +{{ $file := .Get "file" }} +{{ $notebook := .Get "notebook" }} +{{ with (.Get "language") }} +
    +
    + {{ with $file }} + {{ with $notebook }} + + + + {{ end }} + + + + + + + {{ else }} + + + + {{ end }} {{ highlight $content . "" }} - {{ else }} - {{ highlight $content "" "" }} - {{ end }} +
    {{ else }} - {{ with (.Get 0) }} -
    - {{ highlight $content . "" }} + {{ with $language }} +
    +
    + + + + {{ highlight $content . "" }} +
    {{ else }} - {{ highlight $content "" "" }} +
    +
    + + + + {{ highlight $content "" "" }} +
    +
    {{ end }} {{ end }} diff --git a/website/www/site/static/images/logos/runners/gray-logos-1.svg b/website/www/site/static/images/logos/runners/gray-logos-1.svg new file mode 100644 index 000000000000..3b81db34b391 --- /dev/null +++ b/website/www/site/static/images/logos/runners/gray-logos-1.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + diff --git a/website/www/site/static/images/logos/sdks/java-gray.svg b/website/www/site/static/images/logos/sdks/java-gray.svg new file mode 100644 index 000000000000..4fe438f0a632 --- /dev/null +++ b/website/www/site/static/images/logos/sdks/java-gray.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + diff --git a/website/www/site/static/images/logos/sdks/python-gray.svg b/website/www/site/static/images/logos/sdks/python-gray.svg new file mode 100644 index 000000000000..8a1c59c83955 --- /dev/null +++ b/website/www/site/static/images/logos/sdks/python-gray.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + diff --git a/website/www/site/static/js/copy-to-clipboard.js b/website/www/site/static/js/copy-to-clipboard.js new file mode 100644 index 000000000000..fad14d02e82c --- /dev/null +++ b/website/www/site/static/js/copy-to-clipboard.js @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 (the 'License'); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +$(document).ready(function() { + function copy() { + $(".copy").click(function(){ + var text=$(this).siblings()[$(this).siblings().length-1].childNodes[0].innerHTML; + const el=document.createElement('textarea'); + el.value=text;document.body.appendChild(el); + el.select();document.execCommand('copy'); + document.body.removeChild(el); + alert('copied to clipboard'); + }) + $(".just-copy").click(function(){ + var text=$(this).parent().siblings()[0].innerHTML; + const el=document.createElement('textarea'); + el.value=text;document.body.appendChild(el); + el.select();document.execCommand('copy'); + document.body.removeChild(el); + alert('copied to clipboard'); + }) + } + let code = document.querySelectorAll('pre'), + copyIcon = document.createElement('span'); + copyIcon.innerHTML = ''; + + code.forEach((hl) => { + if( !hl.parentElement.classList.contains('code-snippet') && !hl.parentElement.classList.contains('highlight')) { + hl.prepend(copyIcon.cloneNode([true])); + } + }) + copy(); +}); diff --git a/website/www/site/static/js/expandable-list.js b/website/www/site/static/js/expandable-list.js new file mode 100644 index 000000000000..740ad8fd1405 --- /dev/null +++ b/website/www/site/static/js/expandable-list.js @@ -0,0 +1,38 @@ +// Licensed under the Apache License, Version 2.0 (the 'License'); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +$(document).ready(function() { + function expandable(){ + function stop(){ + var w = $(window).width(); + $('.arrow-list-header').click(function(e){ + if (w <= 768) { + e.preventDefault(); + e.stopPropagation(); + } + }) + } + } + $(window).resize(function(){ + stop(); + }); + function rotate() { + $(".arrow-list-header").click(function(){ + $(this).find('figure').toggleClass('rotate'); + }); + $('#apache-dropdown').click(function(){ + $(this).find('span').toggleClass('rotate'); + }); + } + rotate(); + expandable(); +});