diff --git a/Makefile b/Makefile index 95d6f95f7b..52085c1ffe 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,11 @@ client: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-client ./components/client cluster: +ifeq ($(UI),1) + $(GOBUILD) -ldflags '$(LDFLAGS)' -tags ui_server -o bin/tiup-cluster ./components/cluster +else $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-cluster ./components/cluster +endif dm: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-dm ./components/dm @@ -59,6 +63,10 @@ doc: errdoc: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-errdoc ./components/errdoc +embed_cluster_ui: + cd cluster-ui && yarn && yarn build + tools/embed_assets/embed_cluster_ui_assets.sh + server: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-server ./server diff --git a/cluster-ui/.gitignore b/cluster-ui/.gitignore new file mode 100644 index 0000000000..4d29575de8 --- /dev/null +++ b/cluster-ui/.gitignore @@ -0,0 +1,23 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# production +/build + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/cluster-ui/CHANGELOG.md b/cluster-ui/CHANGELOG.md new file mode 100644 index 0000000000..aec80bae7f --- /dev/null +++ b/cluster-ui/CHANGELOG.md @@ -0,0 +1,41 @@ +# TiUP Cluster UI Changelog + +## 2020.11.13 + +- Add audit for deploy/destroy/start/stop/scale_in/scale_out operations + +## 2020.11.04 + +- Support config arch for machine + +## 2020.11.03 + +- Use default "root" as the machine login user name if leave it empty +- Set default labels values if leave them empty +- Add confirmation prompt when starting to deploy + +## 2020.10.26 + +- Enable manually edit the topo yaml configuration +- Support config the numa_node option for TiDB/TiKV/PD/TiFlash + +## 2020.10.21 + +- Skip check TiKV location labels when deploying or scaling out to enable deploy multiple TiKV instances in a same host + +## 2020.10.16 + +- Add data management and db users management features by embeding TiDB dashboard +- Add full TiDB dashboard features entry + +## 2020.10.15 + +- Support to modify mirror address + +## 2020.10.14 + +- Support to modify cluster configuration by embeding TiDB dashboard + +## 2020.09.16 + +- Enable select TiDB v4.0.6 to deploy, support type any TiDB version manually diff --git a/cluster-ui/README.md b/cluster-ui/README.md new file mode 100644 index 0000000000..f1dd478f83 --- /dev/null +++ b/cluster-ui/README.md @@ -0,0 +1,29 @@ +# Cluster Web UI + +This is the web ui for tiup-cluster command. + +## How to Run + +### Release Mode + +```shell +$ cd tiup +$ make embed_cluster_ui +$ UI=1 make +$ bin/tiup-cluster --ui +``` + +Then access `http://127.0.0.1:8080` in the browser. + +### Develop Mode + +```shell +$ cd tiup +$ make +$ bin/tiup-cluster --ui +# a new tab +$ cd cluster-ui +$ yarn && yarn start +``` + +It will auto open `http://127.0.0.1:3000/tiup` in the browser. diff --git a/cluster-ui/config-overrides.js b/cluster-ui/config-overrides.js new file mode 100644 index 0000000000..97b1433cdf --- /dev/null +++ b/cluster-ui/config-overrides.js @@ -0,0 +1,36 @@ +const { override, addLessLoader } = require('customize-cra') +const { alias, configPaths } = require('react-app-rewire-alias') + +const addAlias = () => (config) => { + alias({ + ...configPaths('tsconfig.paths.json'), + })(config) + return config +} + +const configEslint = () => (config) => { + const eslintRule = config.module.rules.filter( + (r) => + r.use && r.use.some((u) => u.options && u.options.useEslintrc !== void 0) + )[0] + const options = eslintRule.use[0].options + // options.ignore = true + // options.ignorePattern = 'lib/client/api/*.ts' + + // To close "The href attribute is required for an anchor to be keyboard accessible" warning + options.baseConfig.rules = { + 'jsx-a11y/anchor-is-valid': 'off', + } + return config +} + +module.exports = override( + addLessLoader({ + lessOptions: { + javascriptEnabled: true, + modifyVars: { '@primary-color': '#3351ff' }, + }, + }), + addAlias(), + configEslint() +) diff --git a/cluster-ui/package.json b/cluster-ui/package.json new file mode 100644 index 0000000000..e1f2a22f86 --- /dev/null +++ b/cluster-ui/package.json @@ -0,0 +1,53 @@ +{ + "name": "tiup-ui", + "version": "0.1.0", + "private": true, + "homepage": "/tiup", + "dependencies": { + "@ant-design/icons": "^4.2.2", + "@testing-library/jest-dom": "^4.2.4", + "@testing-library/react": "^9.3.2", + "@testing-library/user-event": "^7.1.2", + "@types/jest": "^24.0.0", + "@types/node": "^12.0.0", + "@types/react": "^16.9.0", + "@types/react-dom": "^16.9.0", + "@types/uniqid": "^5.2.0", + "ahooks": "^2.5.0", + "antd": "^4.5.3", + "history": "^5.0.0", + "react": "^16.13.1", + "react-dom": "^16.13.1", + "react-router-dom": "^6.0.0-beta.0", + "react-scripts": "3.4.2", + "typescript": "4.0.2", + "uniqid": "^5.2.0", + "yaml": "^1.10.0" + }, + "scripts": { + "start": "react-app-rewired start", + "build": "react-app-rewired build", + "test": "react-app-rewired test" + }, + "eslintConfig": { + "extends": "react-app" + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "devDependencies": { + "customize-cra": "^1.0.0", + "less-loader": "^6.2.0", + "react-app-rewire-alias": "^0.1.6", + "react-app-rewired": "^2.1.6" + } +} diff --git a/cluster-ui/public/favicon.ico b/cluster-ui/public/favicon.ico new file mode 100644 index 0000000000..bcd5dfd67c Binary files /dev/null and b/cluster-ui/public/favicon.ico differ diff --git a/cluster-ui/public/index.html b/cluster-ui/public/index.html new file mode 100644 index 0000000000..d695588d73 --- /dev/null +++ b/cluster-ui/public/index.html @@ -0,0 +1,98 @@ + + + + + + + + TiUP + + + + + +
+
+
+ + diff --git a/cluster-ui/public/logo192.png b/cluster-ui/public/logo192.png new file mode 100644 index 0000000000..fc44b0a379 Binary files /dev/null and b/cluster-ui/public/logo192.png differ diff --git a/cluster-ui/public/logo512.png b/cluster-ui/public/logo512.png new file mode 100644 index 0000000000..a4e47a6545 Binary files /dev/null and b/cluster-ui/public/logo512.png differ diff --git a/cluster-ui/public/manifest.json b/cluster-ui/public/manifest.json new file mode 100644 index 0000000000..080d6c77ac --- /dev/null +++ b/cluster-ui/public/manifest.json @@ -0,0 +1,25 @@ +{ + "short_name": "React App", + "name": "Create React App Sample", + "icons": [ + { + "src": "favicon.ico", + "sizes": "64x64 32x32 24x24 16x16", + "type": "image/x-icon" + }, + { + "src": "logo192.png", + "type": "image/png", + "sizes": "192x192" + }, + { + "src": "logo512.png", + "type": "image/png", + "sizes": "512x512" + } + ], + "start_url": ".", + "display": "standalone", + "theme_color": "#000000", + "background_color": "#ffffff" +} diff --git a/cluster-ui/public/robots.txt b/cluster-ui/public/robots.txt new file mode 100644 index 0000000000..e9e57dc4d4 --- /dev/null +++ b/cluster-ui/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/cluster-ui/src/App.less b/cluster-ui/src/App.less new file mode 100644 index 0000000000..ce63c10da7 --- /dev/null +++ b/cluster-ui/src/App.less @@ -0,0 +1 @@ +@import '~antd/dist/antd.less'; diff --git a/cluster-ui/src/App.tsx b/cluster-ui/src/App.tsx new file mode 100644 index 0000000000..44f2d952e0 --- /dev/null +++ b/cluster-ui/src/App.tsx @@ -0,0 +1,41 @@ +import React from 'react' +import { HashRouter as Router, Routes, Route, Navigate } from 'react-router-dom' + +import StatusPage from '_pages/Status' +import HomePage from '_pages/Home' +import MachinesPage from '_pages/Machines' +import DeploymentPage from '_pages/Deployment' +import ClustersPage from '_pages/Clusters' +import ClusterDetailPage from '_pages/Clusters/ClusterDetail' +import ClusterScaleOutPage from '_pages/Clusters/ClusterScaleOut' +import DashboardPortalPage from '_pages/Clusters/DashboardPortal' +import SettingPage from '_pages/Setting' + +import './App.less' + +function App() { + return ( + + + } /> + + }> + } /> + }> + } /> + } + /> + } /> + + } /> + } /> + } /> + + + + ) +} + +export default App diff --git a/cluster-ui/src/apis/api.ts b/cluster-ui/src/apis/api.ts new file mode 100644 index 0000000000..380dc418e2 --- /dev/null +++ b/cluster-ui/src/apis/api.ts @@ -0,0 +1,67 @@ +import request from './request' + +const API_URL = + process.env.NODE_ENV === 'production' ? '/api' : 'http://127.0.0.1:8080/api' + +function fullUrl(path: string): string { + return `${API_URL}/${path}` +} + +//////////////////// + +export function deployCluster(deployment: any) { + return request(fullUrl('deploy'), 'POST', deployment) +} + +export function getStatus() { + return request(fullUrl('status')) +} + +export function getClusterList() { + return request(fullUrl('clusters')) +} + +export function deleteCluster(clusterName: string) { + return request(fullUrl(`clusters/${clusterName}`), 'DELETE') +} + +export function getClusterTopo(clusterName: string) { + return request(fullUrl(`clusters/${clusterName}`)) +} + +export function startCluster(clusterName: string) { + return request(fullUrl(`clusters/${clusterName}/start`), 'POST') +} + +export function stopCluster(clusterName: string) { + return request(fullUrl(`clusters/${clusterName}/stop`), 'POST') +} + +export function scaleInCluster( + clusterName: string, + scaleInOpts: { nodes: string[]; force: boolean } +) { + return request( + fullUrl(`clusters/${clusterName}/scale_in`), + 'POST', + scaleInOpts + ) +} + +export function scaleOutCluster(clusterName: string, scaleOutOpts: any) { + return request( + fullUrl(`clusters/${clusterName}/scale_out`), + 'POST', + scaleOutOpts + ) +} + +export function getMirrorAddress() { + return request(fullUrl(`mirror`)) +} + +export function setMirrorAddress(newAddress: string) { + return request(fullUrl(`mirror`), 'POST', { + mirror_address: newAddress, + }) +} diff --git a/cluster-ui/src/apis/index.ts b/cluster-ui/src/apis/index.ts new file mode 100644 index 0000000000..2268b2b95c --- /dev/null +++ b/cluster-ui/src/apis/index.ts @@ -0,0 +1,3 @@ +export * from './request' +export { default as request } from './request' +export * from './api' diff --git a/cluster-ui/src/apis/request.ts b/cluster-ui/src/apis/request.ts new file mode 100644 index 0000000000..313a4a7b0c --- /dev/null +++ b/cluster-ui/src/apis/request.ts @@ -0,0 +1,51 @@ +export type ResError = Error & { + response?: any +} + +export default function request( + url: string, + method?: 'GET' | 'POST' | 'PUT' | 'DELETE', + body?: object, + options?: RequestInit +) { + const opts: RequestInit = { + ...options, + method: method || 'GET', + headers: { + ...options?.headers, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + } + if (body) { + opts.body = JSON.stringify(body) + } + return doFetch(url, opts) +} + +function doFetch(url: string, options: RequestInit) { + return fetch(url, options) + .then(parseResponse) + .then((data) => ({ data, err: undefined })) + .catch((err) => ({ data: undefined, err })) +} + +function parseResponse(response: Response) { + if (response.status === 204) { + return {} as any + } else if (response.status >= 200 && response.status < 300) { + return response.json() + } else { + let errMsg = response.statusText + return response + .json() + .then((resData: any) => { + errMsg = resData.msg || resData.message || response.statusText + }) + .finally(() => { + const error: ResError = new Error(errMsg) + error.response = response + throw error + }) + } +} diff --git a/cluster-ui/src/components/Root.tsx b/cluster-ui/src/components/Root.tsx new file mode 100644 index 0000000000..dd63dabde5 --- /dev/null +++ b/cluster-ui/src/components/Root.tsx @@ -0,0 +1,5 @@ +import React, { FC } from 'react' + +const Root: FC = ({ children }) =>
{children}
+ +export default Root diff --git a/cluster-ui/src/components/index.ts b/cluster-ui/src/components/index.ts new file mode 100644 index 0000000000..098334a73c --- /dev/null +++ b/cluster-ui/src/components/index.ts @@ -0,0 +1 @@ +export { default as Root } from './Root' diff --git a/cluster-ui/src/hooks/index.ts b/cluster-ui/src/hooks/index.ts new file mode 100644 index 0000000000..5e54f24080 --- /dev/null +++ b/cluster-ui/src/hooks/index.ts @@ -0,0 +1,5 @@ +export * from './useMachines' +export * from './useComps' +export * from './useGlobalLoginOptions' +export * from './useGlobalDir' +export * from './useQueryParams' diff --git a/cluster-ui/src/hooks/useComps.ts b/cluster-ui/src/hooks/useComps.ts new file mode 100644 index 0000000000..bb0fba34c0 --- /dev/null +++ b/cluster-ui/src/hooks/useComps.ts @@ -0,0 +1,21 @@ +import { useLocalStorageState } from 'ahooks' +import { useMemo } from 'react' + +import { BaseComp, CompMap } from '_types' + +export function useComps() { + const [compObjs, setCompObjs] = useLocalStorageState( + 'components', + {} + ) + + const comps = useMemo(() => { + let _comps: CompMap = {} + Object.keys(compObjs).forEach((k) => { + _comps[k] = BaseComp.deSerial(compObjs[k]) + }) + return _comps + }, [compObjs]) + + return { comps, setCompObjs } +} diff --git a/cluster-ui/src/hooks/useGlobalDir.ts b/cluster-ui/src/hooks/useGlobalDir.ts new file mode 100644 index 0000000000..b283085ffd --- /dev/null +++ b/cluster-ui/src/hooks/useGlobalDir.ts @@ -0,0 +1,20 @@ +import { useLocalStorageState } from 'ahooks' + +import { GlobalDir, IGlobalDir } from '_types' +import { useMemo } from 'react' + +export function useGlobalDir() { + const [globalDirObj, setGlobalDirObj] = useLocalStorageState( + 'global_dir', + {} + ) + + const globalDir = useMemo(() => { + return GlobalDir.deSerial(globalDirObj) + }, [globalDirObj]) + + return { + globalDir, + setGlobalDirObj, + } +} diff --git a/cluster-ui/src/hooks/useGlobalLoginOptions.ts b/cluster-ui/src/hooks/useGlobalLoginOptions.ts new file mode 100644 index 0000000000..751792ff12 --- /dev/null +++ b/cluster-ui/src/hooks/useGlobalLoginOptions.ts @@ -0,0 +1,14 @@ +import { useLocalStorageState } from 'ahooks' + +import { IGlobalLoginOptions } from '_types' + +export function useGlobalLoginOptions() { + const [globalLoginOptions, setGlobalLoginOptions] = useLocalStorageState< + IGlobalLoginOptions + >('global_login_options', {}) + + return { + globalLoginOptions, + setGlobalLoginOptions, + } +} diff --git a/cluster-ui/src/hooks/useMachines.ts b/cluster-ui/src/hooks/useMachines.ts new file mode 100644 index 0000000000..fa40f6c6f4 --- /dev/null +++ b/cluster-ui/src/hooks/useMachines.ts @@ -0,0 +1,21 @@ +import { useLocalStorageState } from 'ahooks' +import { useMemo } from 'react' + +import { MachineMap, Machine } from '_types' + +export function useMachines() { + const [machineObjs, setMachineObjs] = useLocalStorageState( + 'machines', + {} + ) + + const machines = useMemo(() => { + let _machines: MachineMap = {} + Object.keys(machineObjs).forEach((k) => { + _machines[k] = Machine.deSerial(machineObjs[k]) + }) + return _machines + }, [machineObjs]) + + return { machines, setMachineObjs } +} diff --git a/cluster-ui/src/hooks/useQueryParams.ts b/cluster-ui/src/hooks/useQueryParams.ts new file mode 100644 index 0000000000..81a32b9015 --- /dev/null +++ b/cluster-ui/src/hooks/useQueryParams.ts @@ -0,0 +1,17 @@ +import { useMemo } from 'react' +import { useLocation } from 'react-router' + +export function useQueryParams() { + const { search } = useLocation() + + const params = useMemo(() => { + const searchParams = new URLSearchParams(search) + let _params: { [k: string]: any } = {} + for (const [k, v] of searchParams) { + _params[k] = v + } + return _params + }, [search]) + + return params +} diff --git a/cluster-ui/src/index.css b/cluster-ui/src/index.css new file mode 100644 index 0000000000..ec2585e8c0 --- /dev/null +++ b/cluster-ui/src/index.css @@ -0,0 +1,13 @@ +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', + 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', + sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', + monospace; +} diff --git a/cluster-ui/src/index.tsx b/cluster-ui/src/index.tsx new file mode 100644 index 0000000000..9cc87711a7 --- /dev/null +++ b/cluster-ui/src/index.tsx @@ -0,0 +1,12 @@ +import React from 'react' +import ReactDOM from 'react-dom' + +import App from './App' +import './index.css' + +ReactDOM.render( + + + , + document.getElementById('root') +) diff --git a/cluster-ui/src/pages/Clusters/ClusterDetail.tsx b/cluster-ui/src/pages/Clusters/ClusterDetail.tsx new file mode 100644 index 0000000000..a4b3cffd47 --- /dev/null +++ b/cluster-ui/src/pages/Clusters/ClusterDetail.tsx @@ -0,0 +1,275 @@ +import React, { useMemo, useState, useCallback, useEffect, useRef } from 'react' +import { useParams, useNavigate } from 'react-router-dom' +import { Space, Button, Modal, Table, Popconfirm, Divider } from 'antd' +import { ExclamationCircleOutlined } from '@ant-design/icons' +import { useSessionStorageState, useLocalStorageState } from 'ahooks' + +import { + deleteCluster, + getClusterTopo, + startCluster, + stopCluster, + scaleInCluster, +} from '_apis' +import { Root } from '_components' +import { useComps } from '_hooks' +import { ICluster, IClusterInstInfo } from '_types' + +function ClusterDetailPage() { + const refIframe = useRef(null) + + const navigate = useNavigate() + const [clustersList] = useSessionStorageState('clusters', []) + const { clusterName } = useParams() + const cluster = useMemo( + () => clustersList.find((el) => el.name === clusterName), + [clustersList, clusterName] + ) + + const [clusterInstInfos, setClusterInstInfos] = useSessionStorageState< + IClusterInstInfo[] + >(`${clusterName}_cluster_topo`, []) + + const dashboardPD = useMemo(() => { + return clusterInstInfos.find( + (el) => + el.role === 'pd' && + el.status.indexOf('UI') !== -1 && + el.status.indexOf('Up') !== -1 + ) + }, [clusterInstInfos]) + + const [curScaleOutNodes] = useLocalStorageState<{ + cluster_name: string + scale_out_nodes: any[] + }>('cur_scale_out_nodes', { cluster_name: '', scale_out_nodes: [] }) + const { comps, setCompObjs } = useComps() + + const [loadingTopo, setLoadingTopo] = useState(false) + + const handleScaleInCluster = useCallback( + (node: IClusterInstInfo) => { + const lowerStatus = node.status.toLowerCase() + const force = + lowerStatus.indexOf('down') !== -1 || + lowerStatus.indexOf('inactive') !== -1 + + scaleInCluster(clusterName, { + nodes: [node.id], + force, + }) + navigate('/status') + }, + [navigate, clusterName] + ) + + const columns = useMemo(() => { + const _columns = [ + 'ID', + 'Role', + 'Host', + 'Ports', + 'OS_Arch', + 'Status', + 'Data_Dir', + 'Deploy_Dir', + ].map((title) => ({ + title, + key: title.toLowerCase(), + dataIndex: title.toLowerCase(), + fixed: title === 'ID', + })) + _columns.push({ + title: '操作', + key: 'action', + width: 100, + fixed: 'right', + render: (text: any, rec: IClusterInstInfo) => { + if (rec.status.toLowerCase().indexOf('offline') !== -1) { + return null + } + return ( + handleScaleInCluster(rec)} + okText="下线" + cancelText="取消" + > + 缩容 + + ) + }, + } as any) + return _columns + }, [handleScaleInCluster]) + + useEffect(() => { + setLoadingTopo(true) + getClusterTopo(clusterName).then(({ data, err }) => { + setLoadingTopo(false) + if (data !== undefined) { + setClusterInstInfos(data) + updateLocalTopo(data) + } + }) + // eslint-disable-next-line + }, []) + + const [dashboardToken] = useSessionStorageState( + `${clusterName}_dashboard_token`, + '' + ) + + function updateLocalTopo(clusters: IClusterInstInfo[]) { + if ( + curScaleOutNodes.cluster_name !== clusterName || + curScaleOutNodes.scale_out_nodes.length === 0 + ) { + return + } + let newComps = { ...comps } + for (const n of curScaleOutNodes.scale_out_nodes) { + const exist = clusters.find((el) => el.id === n.node) + if (exist && newComps[n.id]) { + newComps[n.id].for_scale_out = false + } + } + setCompObjs(newComps) + } + + function destroyCluster() { + deleteCluster(clusterName) + navigate('/status') + } + + function handleDestroyCluster() { + Modal.confirm({ + title: `销毁 ${cluster?.name} 集群`, + icon: , + content: '你确定要销毁这个集群吗？所有数据都会清除，操作不可回滚！', + okText: '销毁', + cancelText: '取消', + okButtonProps: { danger: true }, + onOk: () => destroyCluster(), + }) + } + + function handleStartCluster() { + startCluster(clusterName) + navigate('/status') + } + + function handleStopCluster() { + stopCluster(clusterName) + navigate('/status') + } + + function handleScaleOutCluster() { + navigate(`/clusters/${clusterName}/scaleout`) + } + + function handleOpenDashboard(targetFeature: string) { + if (dashboardPD === undefined) { + Modal.error({ + title: '没有找到相应的 PD 节点', + content: + '请检查相应的 PD 节点是否工作正常，如果集群未启动，请先启动集群。', + }) + return + } + if (targetFeature === 'full') { + refIframe.current?.contentWindow?.postMessage( + { + token: dashboardToken, + lang: 'zh', + hideNav: false, + redirectPath: '/diagnose', + }, + '*' + ) + window.open(`http://${dashboardPD.id}/dashboard/`, '_blank') + return + } + + navigate( + `/clusters/${clusterName}/dashboard?pd=${dashboardPD.id}&tidb_version=${cluster?.version}&target=${targetFeature}` + ) + } + + return ( + +
+ + + + + + + + + + + + + + + +
+ + {cluster && ( +
+ {/*

Name: {cluster.name}

+

User: {cluster.user}

*/} +

Version: {cluster.version}

+ {/*

Path: {cluster.path}

+

PrivateKey: {cluster.private_key}

*/} +
+ )} + + + +