From 0d4cd8b948cabb7addba60bb6ec48999ab30bbd1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 6 Sep 2018 09:14:17 +0000 Subject: [PATCH 1/8] Build out_oms.so and include in docker-cimprov package --- build/Makefile | 829 ++++++++++++------------ installer/datafiles/base_container.data | 5 +- source/code/go/src/plugins/Makefile | 20 + source/code/go/src/plugins/glide.lock | 209 ++++++ source/code/go/src/plugins/glide.yaml | 15 + source/code/go/src/plugins/oms.go | 263 ++++++++ source/code/go/src/plugins/out_oms.go | 65 ++ 7 files changed, 996 insertions(+), 410 deletions(-) create mode 100644 source/code/go/src/plugins/Makefile create mode 100644 source/code/go/src/plugins/glide.lock create mode 100644 source/code/go/src/plugins/glide.yaml create mode 100644 source/code/go/src/plugins/oms.go create mode 100644 source/code/go/src/plugins/out_oms.go diff --git a/build/Makefile b/build/Makefile index 9586c3b23..b5312cfe3 100644 --- a/build/Makefile +++ b/build/Makefile @@ -1,409 +1,420 @@ -# -*- mode: Makefile; -*- -# Copyright (c) Microsoft Corporation - -BASE_DIR := $(subst /build,,$(PWD)) -OMI_ROOT := $(shell cd ../../omi/Unix; pwd -P) -SCXPAL_DIR := $(shell cd ../../pal; pwd -P) - -PF_POSIX := 1 -include $(SCXPAL_DIR)/build/config.mak -include $(BASE_DIR)/build/config.mak -include $(SCXPAL_DIR)/build/Makefile.pal - -ifndef ENABLE_DEBUG -$(error "ENABLE_DEBUG is not set. Please re-run configure") -endif - -# Include the version file -include ../../docker.version - -ifndef CONTAINER_BUILDVERSION_STATUS -$(error "Is docker.version missing? Please re-run configure") -endif - -SOURCE_DIR := $(BASE_DIR)/source/code -TEST_DIR := $(BASE_DIR)/test/code - -PROVIDER_DIR := $(SOURCE_DIR)/providers -PROVIDER_TEST_DIR := $(TEST_DIR)/providers -PAL_INCLUDE_DIR := $(SCXPAL_DIR)/source/code/include -PAL_TESTUTILS_DIR := $(SCXPAL_DIR)/test/code/testutils - -INTERMEDIATE_DIR := $(BASE_DIR)/intermediate/$(BUILD_CONFIGURATION) -INTERMEDIATE_TESTFILES := $(INTERMEDIATE_DIR)/testfiles -TARGET_DIR := $(BASE_DIR)/target/$(BUILD_CONFIGURATION) -PROVIDER_LIBRARY := $(INTERMEDIATE_DIR)/libcontainer.so - -INSTALLER_TMPDIR := $(INTERMEDIATE_DIR)/installer_tmp - -# Include files - -INCLUDE_DEFINES := $(INTERMEDIATE_DIR)/defines.h - -# Compiler flags - -OMI_INCLUDE_FLAGS := -I$(OMI_ROOT)/output/include -PROVIDER_INCLUDE_FLAGS := -I$(PAL_INCLUDE_DIR) -I$(INTERMEDIATE_DIR) - -PROVIDER_TEST_INCLUDE_FLAGS := -Wmissing-include-dirs -Wno-non-virtual-dtor -I$(SCXPAL_DIR)/source/code/include -I$(INTERMEDIATE_DIR) -I$(SCXPAL_DIR)/test/ext/include -I$(OMI_ROOT)/output/include -I$(OMI_ROOT) -I$(OMI_ROOT)/common -I$(SCXPAL_DIR)/test/code/include $(PROVIDER_INCLUDE_FLAGS) -I$(PROVIDER_DIR) - -ifeq ($(ENABLE_DEBUG),1) -PROV_DEBUG_FLAGS := -g -endif - -COMPILE_FLAGS := $(PROV_DEBUG_FLAGS) -D_REENTRANT -fstack-protector-all -Wall -fno-nonansi-builtins -Woverloaded-virtual -Wformat -Wformat-security -Wcast-align -Wswitch-enum -Wshadow -Wwrite-strings -Wredundant-decls -Wcast-qual -fPIC -PROVIDER_COMPILE_FLAGS := $(COMPILE_FLAGS) - -LINK_LIBRARIES := -Wl,-rpath=/opt/omi/lib -L$(OMI_ROOT)/output/lib -lmicxx -L$(SCXPAL_TARGET_DIR) -lscxcore -lUtil -lscxassertabort -lrt -luuid -PROVIDER_TEST_LINK_LIBRARIES := -lbase -lpal -L$(SCXPAL_TARGET_DIR) -lscxcore $(SCXPAL_DIR)/test/ext/lib/linux/$(ARCH)/cppunit/libcppunit.a -lpthread -lrt -luuid - -SHARED_FLAGS := -shared - -# Support for installbuilder - -STAGING_DIR := $(INTERMEDIATE_DIR)/staging - -ifeq ($(ULINUX),1) - # For consistency, the architecture should be i686 (for x86) and x86_64 (for x64) - DOCKER_ARCH := $(shell echo $(PF_ARCH) | sed -e 's/x86$$/i686/' -e 's/x64$$/x86_64/') - OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).universal.$(DOCKER_ARCH) -else - PF_DISTRO_LC := $(shell echo $(PF_DISTRO) | tr A-Z a-z) - OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).$(PF_DISTRO_LC).$(PF_MAJOR).$(PF_ARCH) -endif - -ifeq ("$(wildcard /usr/bin/dpkg-deb)","") - DPKG_LOCATION="--DPKG_LOCATION=$(SCXPAL_DIR)/installer/InstallBuilder/tools/bin/dpkg-deb-$(PF_ARCH)" -else - DPKG_LOCATION= -endif - -# Support for src_to_obj handling - -INCLUDES = $(OMI_INCLUDE_FLAGS) $(PROVIDER_INCLUDE_FLAGS) -CFLAGS = $(COMPILE_FLAGS) -CXXFLAGS = $(COMPILE_FLAGS) - -#-------------------------------------------------------------------------------- -# Build targets - -ifeq ($(ULINUX),1) -all : $(OMI_ROOT)/output $(SCXPAL_INTERMEDIATE_DIR) PROVIDER_STATUS $(PROVIDER_LIBRARY) KIT_STATUS kit -else -all : $(OMI_ROOT)/output $(SCXPAL_INTERMEDIATE_DIR) PROVIDER_STATUS $(PROVIDER_LIBRARY) -endif - -clean : - $(RMDIR) $(BASE_DIR)/build/cppunit_result.* $(BASE_DIR)/build/scxtestrunner.log $(BASE_DIR)/installer/intermediate $(BASE_DIR)/intermediate $(BASE_DIR)/target $(PROVIDER_TEST_DIR)/providertestutils.cpp - -find $(BASE_DIR) -name \*~ -exec rm {} \; - -$(RM) $(TEST_DIR)/providers/TestScriptPath.h - -distclean : clean - $(RM) $(BASE_DIR)/build/config.mak - -make -C $(OMI_ROOT) distclean - -make -C $(SCXPAL_DIR)/build distclean - -$(RMDIR) $(OMI_ROOT)/output* - -$(RM) $(SCXPAL_DIR)/build/config.mak - -$(RM) $(SCXPAL_DIR)/build/Makefile.config_cache - -PROVIDER_STATUS: - @echo "========================= Performing Building provider" - -KIT_STATUS: - @echo "========================= Performing Building provider tests" - -#-------------------------------------------------------------------------------- -# OMI build -# -# Build the OMI distribution -# -# Technically, we should go to build OMI all the time. But I'd rather not spend -# the time doing it here EVERY TIME, when we never normally change OMI. This is -# a good tradeoff (build if not built, otherwise assume all is well). -# -# Doing a 'make clean' in OMI directory will force us to rebuild. - -$(OMI_ROOT)/output : $(OMI_ROOT)/output/lib/libmicxx.so - -$(OMI_ROOT)/output/lib/libmicxx.so : - @echo "========================= Performing Building OMI" - make -C $(OMI_ROOT) -ifeq ($(PERFORM_OMI_MAKEINSTALL),1) - make -C $(OMI_ROOT) install -endif - -#-------------------------------------------------------------------------------- -# PAL build -# -# Build the PAL (Platform Abstraction Layer) -# -# Doing a 'make clean' in PAL directory will force us to rebuild. - -$(SCXPAL_INTERMEDIATE_DIR) : - @echo "========================= Performing Building PAL" - make -C $(SCXPAL_DIR)/build - -#================================================================================ -# File depends.h (compiler dependencies) -#================================================================================ - -$(INCLUDE_DEFINES) : $(BASE_DIR)/build/config.mak - -$(MKPATH) $(@D) - @$(ECHO) "Creating $@" - @$(call pf_fwrite,"/*-------------------------------------------------------------------------------", $@) - @$(call pf_fappend," Copyright (C) 2007-2015 Microsoft Corp. ", $@) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"*/ ", $@) - @$(call pf_fappend,"/** ", $@) - @$(call pf_fappend," \file ", $@) - @$(call pf_fappend," ", $@) - @$(call pf_fappend," \brief Auto generated file containing build definitions ", $@) - @$(call pf_fappend," ", $@) - @$(call pf_fappend," \author Automated Build System ", $@) - @$(call pf_fappend," ", $@) - @$(call pf_fappend," DO NOT EDIT THIS FILE! ", $@) - @$(call pf_fappend," DO NOT CHECK IN THIS FILE! ", $@) - @$(call pf_fappend,"*/ ", $@) - @$(call pf_fappend,"/*----------------------------------------------------------------------------*/", $@) - @$(call pf_fappend,"#ifndef DEFINES_H ", $@) - @$(call pf_fappend,"#define DEFINES_H ", $@) - @$(call pf_fappend," ", $@) -ifneq ($(PF_DISTRO),) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef PF_DISTRO_$(PF_DISTRO) ", $@) - @$(call pf_fappend,"#define PF_DISTRO_$(PF_DISTRO) ", $@) - @$(call pf_fappend,"#endif ", $@) -endif -ifneq ($(PF_MAJOR),) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef PF_MAJOR ", $@) - @$(call pf_fappend,"#define PF_MAJOR $(PF_MAJOR) ", $@) - @$(call pf_fappend,"#endif ", $@) -endif -ifneq ($(PF_MINOR),) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef PF_MINOR ", $@) - @$(call pf_fappend,"#define PF_MINOR $(PF_MINOR) ", $@) - @$(call pf_fappend,"#endif ", $@) -endif -ifneq ($(ARCH),) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef $(ARCH) ", $@) - @$(call pf_fappend,"#define $(ARCH) ", $@) - @$(call pf_fappend,"#endif ", $@) -endif -ifeq ($(BUILD_TYPE),Debug) - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef _DEBUG ", $@) - @$(call pf_fappend,"#define _DEBUG ", $@) - @$(call pf_fappend,"#endif ", $@) -else - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#ifndef NDEBUG ", $@) - @$(call pf_fappend,"#define NDEBUG ", $@) - @$(call pf_fappend,"#endif ", $@) -endif - @$(call pf_fappend," ", $@) - @$(call pf_fappend,"#endif /* DEFINES_H */ ", $@) - @$(call pf_fappend,"/*----------------------------E-N-D---O-F---F-I-L-E---------------------------*/", $@) - -#================================================================================ -# Internal functions -#================================================================================ - -# Convert a list of src files with absolute paths under BASE_DIR to corresponding -# object files under intermediate directory -# src_to_obj(list_of_cppfiles) -src_to_obj = $(patsubst $(BASE_DIR)%, $(INTERMEDIATE_DIR)%, $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(1)))) - -# No default rules, please -.SUFFIX: - -# Rule for compiling cpp files in source tree, ouptut in mirrored intermediate dir -$(INTERMEDIATE_DIR)/%.o : $(BASE_DIR)/%.cpp $(INCLUDE_DEFINES) - $(MKPATH) $(@D) - $(CXX) -c $(CXXFLAGS) $(INCLUDES) -I$( $(TEST_DIR)/providers/TestScriptPath.h - -test : TEST_STATUS $(SCXPAL_INTERMEDIATE_DIR) $(INTERMEDIATE_DIR)/testrunner - @echo "========================= Performing container testrun execution" - $(MKPATH) $(INTERMEDIATE_TESTFILES) - $(COPY) $(TEST_DIR)/scripts/createEnv.sh $(TEST_DIR)/scripts/testrun_wrapper $(INTERMEDIATE_TESTFILES) - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(OMI_ROOT)/output/lib; cd $(INTERMEDIATE_TESTFILES); ./createEnv.sh - cd $(INTERMEDIATE_TESTFILES); ./testrun_wrapper $(INTERMEDIATE_DIR) - -#-------------------------------------------------------------------------------- -# Build the distribution kit -# -# Build the packages via installbuilder -# -# While the "formal build" only builds ULINUX, we may build something else for DEV purposes. -# Assume we ALWAYS build DPKG, but only build RPM if --enable-ulinux is speified in configure. - -kit : CONTAINERLIB_FILENAME = libcontainer.so -kit : $(OMI_ROOT)/output $(PROVIDER_LIBRARY) - -ifeq ($(ULINUX),1) - - @echo "========================= Performing Building RPM and DPKG packages" - $(MKPATH) $(INSTALLER_TMPDIR) - sudo $(RMDIR) $(STAGING_DIR) - $(MKPATH) $(INTERMEDIATE_DIR) - python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ - --BASE_DIR=$(BASE_DIR) \ - --TARGET_DIR=$(INTERMEDIATE_DIR) \ - --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ - --STAGING_DIR=$(STAGING_DIR) \ - --BUILD_TYPE=$(BUILD_TYPE) \ - --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ - --PFARCH=$(PF_ARCH) \ - --PFDISTRO=$(PF_DISTRO) \ - --PFMAJOR=$(PF_MAJOR) \ - --PFMINOR=$(PF_MINOR) \ - --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ - --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ - --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ - --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ - --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ - base_container.data linux.data linux_rpm.data - - sudo $(RMDIR) $(STAGING_DIR) - $(MKPATH) $(INTERMEDIATE_DIR) - python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ - --BASE_DIR=$(BASE_DIR) \ - --TARGET_DIR=$(INTERMEDIATE_DIR) \ - --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ - --STAGING_DIR=$(STAGING_DIR) \ - --BUILD_TYPE=$(BUILD_TYPE) \ - --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ - --PFARCH=$(PF_ARCH) \ - --PFDISTRO=$(PF_DISTRO) \ - --PFMAJOR=$(PF_MAJOR) \ - --PFMINOR=$(PF_MINOR) \ - --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ - --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ - --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ - $(DPKG_LOCATION) \ - --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ - --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ - base_container.data linux.data linux_dpkg.data - - # Strip the package extension from the package filename - sed -re 's/.rpm$$|.deb$$//' $(INTERMEDIATE_DIR)/package_filename > $(INTERMEDIATE_DIR)/package_file.tmp; mv $(INTERMEDIATE_DIR)/package_file.tmp $(INTERMEDIATE_DIR)/package_filename - - # Build the tar file containing both .rpm and .deb packages - cd $(INTERMEDIATE_DIR); tar cvf $(OUTPUT_PACKAGE_PREFIX).tar $(OUTPUT_PACKAGE_PREFIX).rpm $(OUTPUT_PACKAGE_PREFIX).deb - - ../installer/bundle/create_bundle.sh $(PF)_$(PF_DISTRO) $(INTERMEDIATE_DIR) $(OUTPUT_PACKAGE_PREFIX) - # Copy the shell bundle to the target directory - $(MKPATH) $(TARGET_DIR) - cd $(INTERMEDIATE_DIR); $(COPY) `cat $(INTERMEDIATE_DIR)/package_filename`.sh $(TARGET_DIR) - -else - - @echo "========================= Performing Building RPM and DPKG packages" - sudo $(RMDIR) $(STAGING_DIR) - $(MKPATH) $(INTERMEDIATE_DIR) - python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ - --BASE_DIR=$(BASE_DIR) \ - --TARGET_DIR=$(INTERMEDIATE_DIR) \ - --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ - --STAGING_DIR=$(STAGING_DIR) \ - --BUILD_TYPE=$(BUILD_TYPE) \ - --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ - --PFARCH=$(PF_ARCH) \ - --PFDISTRO=$(PF_DISTRO) \ - --PFMAJOR=$(PF_MAJOR) \ - --PFMINOR=$(PF_MINOR) \ - --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ - --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ - --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ - $(DPKG_LOCATION) \ - --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ - --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ - base_container.data linux.data linux_dpkg.data - -endif +# -*- mode: Makefile; -*- +# Copyright (c) Microsoft Corporation + +BASE_DIR := $(subst /build,,$(PWD)) +OMI_ROOT := $(shell cd ../../omi/Unix; pwd -P) +SCXPAL_DIR := $(shell cd ../../pal; pwd -P) + +PF_POSIX := 1 +include $(SCXPAL_DIR)/build/config.mak +include $(BASE_DIR)/build/config.mak +include $(SCXPAL_DIR)/build/Makefile.pal + +ifndef ENABLE_DEBUG +$(error "ENABLE_DEBUG is not set. Please re-run configure") +endif + +# Include the version file +include ../../docker.version + +ifndef CONTAINER_BUILDVERSION_STATUS +$(error "Is docker.version missing? Please re-run configure") +endif + +SOURCE_DIR := $(BASE_DIR)/source/code +TEST_DIR := $(BASE_DIR)/test/code + +PROVIDER_DIR := $(SOURCE_DIR)/providers +PROVIDER_TEST_DIR := $(TEST_DIR)/providers +PAL_INCLUDE_DIR := $(SCXPAL_DIR)/source/code/include +PAL_TESTUTILS_DIR := $(SCXPAL_DIR)/test/code/testutils + +INTERMEDIATE_DIR := $(BASE_DIR)/intermediate/$(BUILD_CONFIGURATION) +INTERMEDIATE_TESTFILES := $(INTERMEDIATE_DIR)/testfiles +TARGET_DIR := $(BASE_DIR)/target/$(BUILD_CONFIGURATION) +PROVIDER_LIBRARY := $(INTERMEDIATE_DIR)/libcontainer.so + +INSTALLER_TMPDIR := $(INTERMEDIATE_DIR)/installer_tmp + +# GO Source dir for custom fluent bit plugin +GO_SOURCE_DIR := $(SOURCE_DIR)/go/src/plugins + +# Include files + +INCLUDE_DEFINES := $(INTERMEDIATE_DIR)/defines.h + +# Compiler flags + +OMI_INCLUDE_FLAGS := -I$(OMI_ROOT)/output/include +PROVIDER_INCLUDE_FLAGS := -I$(PAL_INCLUDE_DIR) -I$(INTERMEDIATE_DIR) + +PROVIDER_TEST_INCLUDE_FLAGS := -Wmissing-include-dirs -Wno-non-virtual-dtor -I$(SCXPAL_DIR)/source/code/include -I$(INTERMEDIATE_DIR) -I$(SCXPAL_DIR)/test/ext/include -I$(OMI_ROOT)/output/include -I$(OMI_ROOT) -I$(OMI_ROOT)/common -I$(SCXPAL_DIR)/test/code/include $(PROVIDER_INCLUDE_FLAGS) -I$(PROVIDER_DIR) + +ifeq ($(ENABLE_DEBUG),1) +PROV_DEBUG_FLAGS := -g +endif + +COMPILE_FLAGS := $(PROV_DEBUG_FLAGS) -D_REENTRANT -fstack-protector-all -Wall -fno-nonansi-builtins -Woverloaded-virtual -Wformat -Wformat-security -Wcast-align -Wswitch-enum -Wshadow -Wwrite-strings -Wredundant-decls -Wcast-qual -fPIC +PROVIDER_COMPILE_FLAGS := $(COMPILE_FLAGS) + +LINK_LIBRARIES := -Wl,-rpath=/opt/omi/lib -L$(OMI_ROOT)/output/lib -lmicxx -L$(SCXPAL_TARGET_DIR) -lscxcore -lUtil -lscxassertabort -lrt -luuid +PROVIDER_TEST_LINK_LIBRARIES := -lbase -lpal -L$(SCXPAL_TARGET_DIR) -lscxcore $(SCXPAL_DIR)/test/ext/lib/linux/$(ARCH)/cppunit/libcppunit.a -lpthread -lrt -luuid + +SHARED_FLAGS := -shared + +# Support for installbuilder + +STAGING_DIR := $(INTERMEDIATE_DIR)/staging + +ifeq ($(ULINUX),1) + # For consistency, the architecture should be i686 (for x86) and x86_64 (for x64) + DOCKER_ARCH := $(shell echo $(PF_ARCH) | sed -e 's/x86$$/i686/' -e 's/x64$$/x86_64/') + OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).universal.$(DOCKER_ARCH) +else + PF_DISTRO_LC := $(shell echo $(PF_DISTRO) | tr A-Z a-z) + OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).$(PF_DISTRO_LC).$(PF_MAJOR).$(PF_ARCH) +endif + +ifeq ("$(wildcard /usr/bin/dpkg-deb)","") + DPKG_LOCATION="--DPKG_LOCATION=$(SCXPAL_DIR)/installer/InstallBuilder/tools/bin/dpkg-deb-$(PF_ARCH)" +else + DPKG_LOCATION= +endif + +# Support for src_to_obj handling + +INCLUDES = $(OMI_INCLUDE_FLAGS) $(PROVIDER_INCLUDE_FLAGS) +CFLAGS = $(COMPILE_FLAGS) +CXXFLAGS = $(COMPILE_FLAGS) + +#-------------------------------------------------------------------------------- +# Build targets + +ifeq ($(ULINUX),1) +all : $(OMI_ROOT)/output $(SCXPAL_INTERMEDIATE_DIR) PROVIDER_STATUS $(PROVIDER_LIBRARY) KIT_STATUS kit fluentbitplugin +else +all : $(OMI_ROOT)/output $(SCXPAL_INTERMEDIATE_DIR) PROVIDER_STATUS $(PROVIDER_LIBRARY) fluentbitplugin +endif + +clean : + $(RMDIR) $(BASE_DIR)/build/cppunit_result.* $(BASE_DIR)/build/scxtestrunner.log $(BASE_DIR)/installer/intermediate $(BASE_DIR)/intermediate $(BASE_DIR)/target $(PROVIDER_TEST_DIR)/providertestutils.cpp + -find $(BASE_DIR) -name \*~ -exec rm {} \; + -$(RM) $(TEST_DIR)/providers/TestScriptPath.h + +distclean : clean + $(RM) $(BASE_DIR)/build/config.mak + -make -C $(OMI_ROOT) distclean + -make -C $(SCXPAL_DIR)/build distclean + -$(RMDIR) $(OMI_ROOT)/output* + -$(RM) $(SCXPAL_DIR)/build/config.mak + -$(RM) $(SCXPAL_DIR)/build/Makefile.config_cache + +PROVIDER_STATUS: + @echo "========================= Performing Building provider" + +KIT_STATUS: + @echo "========================= Performing Building provider tests" + +#-------------------------------------------------------------------------------- +# OMI build +# +# Build the OMI distribution +# +# Technically, we should go to build OMI all the time. But I'd rather not spend +# the time doing it here EVERY TIME, when we never normally change OMI. This is +# a good tradeoff (build if not built, otherwise assume all is well). +# +# Doing a 'make clean' in OMI directory will force us to rebuild. + +$(OMI_ROOT)/output : $(OMI_ROOT)/output/lib/libmicxx.so + +$(OMI_ROOT)/output/lib/libmicxx.so : + @echo "========================= Performing Building OMI" + make -C $(OMI_ROOT) +ifeq ($(PERFORM_OMI_MAKEINSTALL),1) + make -C $(OMI_ROOT) install +endif + +#--------------------------------------------------------------------------------- +# fluentbit go plugin build. This is required to send container logs to ODS endpoint +# +fluentbitplugin : + @echo "========================= Building fluentbit out_oms go plugin for logs" + make -C $(GO_SOURCE_DIR) fbplugin + $(COPY) $(GO_SOURCE_DIR)/out_oms.so $(INTERMEDIATE_DIR) + +#-------------------------------------------------------------------------------- +# PAL build +# +# Build the PAL (Platform Abstraction Layer) +# +# Doing a 'make clean' in PAL directory will force us to rebuild. + +$(SCXPAL_INTERMEDIATE_DIR) : + @echo "========================= Performing Building PAL" + make -C $(SCXPAL_DIR)/build + +#================================================================================ +# File depends.h (compiler dependencies) +#================================================================================ + +$(INCLUDE_DEFINES) : $(BASE_DIR)/build/config.mak + -$(MKPATH) $(@D) + @$(ECHO) "Creating $@" + @$(call pf_fwrite,"/*-------------------------------------------------------------------------------", $@) + @$(call pf_fappend," Copyright (C) 2007-2015 Microsoft Corp. ", $@) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"*/ ", $@) + @$(call pf_fappend,"/** ", $@) + @$(call pf_fappend," \file ", $@) + @$(call pf_fappend," ", $@) + @$(call pf_fappend," \brief Auto generated file containing build definitions ", $@) + @$(call pf_fappend," ", $@) + @$(call pf_fappend," \author Automated Build System ", $@) + @$(call pf_fappend," ", $@) + @$(call pf_fappend," DO NOT EDIT THIS FILE! ", $@) + @$(call pf_fappend," DO NOT CHECK IN THIS FILE! ", $@) + @$(call pf_fappend,"*/ ", $@) + @$(call pf_fappend,"/*----------------------------------------------------------------------------*/", $@) + @$(call pf_fappend,"#ifndef DEFINES_H ", $@) + @$(call pf_fappend,"#define DEFINES_H ", $@) + @$(call pf_fappend," ", $@) +ifneq ($(PF_DISTRO),) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef PF_DISTRO_$(PF_DISTRO) ", $@) + @$(call pf_fappend,"#define PF_DISTRO_$(PF_DISTRO) ", $@) + @$(call pf_fappend,"#endif ", $@) +endif +ifneq ($(PF_MAJOR),) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef PF_MAJOR ", $@) + @$(call pf_fappend,"#define PF_MAJOR $(PF_MAJOR) ", $@) + @$(call pf_fappend,"#endif ", $@) +endif +ifneq ($(PF_MINOR),) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef PF_MINOR ", $@) + @$(call pf_fappend,"#define PF_MINOR $(PF_MINOR) ", $@) + @$(call pf_fappend,"#endif ", $@) +endif +ifneq ($(ARCH),) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef $(ARCH) ", $@) + @$(call pf_fappend,"#define $(ARCH) ", $@) + @$(call pf_fappend,"#endif ", $@) +endif +ifeq ($(BUILD_TYPE),Debug) + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef _DEBUG ", $@) + @$(call pf_fappend,"#define _DEBUG ", $@) + @$(call pf_fappend,"#endif ", $@) +else + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#ifndef NDEBUG ", $@) + @$(call pf_fappend,"#define NDEBUG ", $@) + @$(call pf_fappend,"#endif ", $@) +endif + @$(call pf_fappend," ", $@) + @$(call pf_fappend,"#endif /* DEFINES_H */ ", $@) + @$(call pf_fappend,"/*----------------------------E-N-D---O-F---F-I-L-E---------------------------*/", $@) + +#================================================================================ +# Internal functions +#================================================================================ + +# Convert a list of src files with absolute paths under BASE_DIR to corresponding +# object files under intermediate directory +# src_to_obj(list_of_cppfiles) +src_to_obj = $(patsubst $(BASE_DIR)%, $(INTERMEDIATE_DIR)%, $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(1)))) + +# No default rules, please +.SUFFIX: + +# Rule for compiling cpp files in source tree, ouptut in mirrored intermediate dir +$(INTERMEDIATE_DIR)/%.o : $(BASE_DIR)/%.cpp $(INCLUDE_DEFINES) + $(MKPATH) $(@D) + $(CXX) -c $(CXXFLAGS) $(INCLUDES) -I$( $(TEST_DIR)/providers/TestScriptPath.h + +test : TEST_STATUS $(SCXPAL_INTERMEDIATE_DIR) $(INTERMEDIATE_DIR)/testrunner + @echo "========================= Performing container testrun execution" + $(MKPATH) $(INTERMEDIATE_TESTFILES) + $(COPY) $(TEST_DIR)/scripts/createEnv.sh $(TEST_DIR)/scripts/testrun_wrapper $(INTERMEDIATE_TESTFILES) + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(OMI_ROOT)/output/lib; cd $(INTERMEDIATE_TESTFILES); ./createEnv.sh + cd $(INTERMEDIATE_TESTFILES); ./testrun_wrapper $(INTERMEDIATE_DIR) + +#-------------------------------------------------------------------------------- +# Build the distribution kit +# +# Build the packages via installbuilder +# +# While the "formal build" only builds ULINUX, we may build something else for DEV purposes. +# Assume we ALWAYS build DPKG, but only build RPM if --enable-ulinux is speified in configure. + +kit : CONTAINERLIB_FILENAME = libcontainer.so +kit : $(OMI_ROOT)/output $(PROVIDER_LIBRARY) fluentbitplugin + +ifeq ($(ULINUX),1) + + @echo "========================= Performing Building RPM and DPKG packages" + $(MKPATH) $(INSTALLER_TMPDIR) + sudo $(RMDIR) $(STAGING_DIR) + $(MKPATH) $(INTERMEDIATE_DIR) + python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ + --BASE_DIR=$(BASE_DIR) \ + --TARGET_DIR=$(INTERMEDIATE_DIR) \ + --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ + --STAGING_DIR=$(STAGING_DIR) \ + --BUILD_TYPE=$(BUILD_TYPE) \ + --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ + --PFARCH=$(PF_ARCH) \ + --PFDISTRO=$(PF_DISTRO) \ + --PFMAJOR=$(PF_MAJOR) \ + --PFMINOR=$(PF_MINOR) \ + --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ + --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ + --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ + --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ + --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ + base_container.data linux.data linux_rpm.data + + sudo $(RMDIR) $(STAGING_DIR) + $(MKPATH) $(INTERMEDIATE_DIR) + python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ + --BASE_DIR=$(BASE_DIR) \ + --TARGET_DIR=$(INTERMEDIATE_DIR) \ + --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ + --STAGING_DIR=$(STAGING_DIR) \ + --BUILD_TYPE=$(BUILD_TYPE) \ + --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ + --PFARCH=$(PF_ARCH) \ + --PFDISTRO=$(PF_DISTRO) \ + --PFMAJOR=$(PF_MAJOR) \ + --PFMINOR=$(PF_MINOR) \ + --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ + --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ + --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ + $(DPKG_LOCATION) \ + --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ + --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ + base_container.data linux.data linux_dpkg.data + + # Strip the package extension from the package filename + sed -re 's/.rpm$$|.deb$$//' $(INTERMEDIATE_DIR)/package_filename > $(INTERMEDIATE_DIR)/package_file.tmp; mv $(INTERMEDIATE_DIR)/package_file.tmp $(INTERMEDIATE_DIR)/package_filename + + # Build the tar file containing both .rpm and .deb packages + cd $(INTERMEDIATE_DIR); tar cvf $(OUTPUT_PACKAGE_PREFIX).tar $(OUTPUT_PACKAGE_PREFIX).rpm $(OUTPUT_PACKAGE_PREFIX).deb + + ../installer/bundle/create_bundle.sh $(PF)_$(PF_DISTRO) $(INTERMEDIATE_DIR) $(OUTPUT_PACKAGE_PREFIX) + # Copy the shell bundle to the target directory + $(MKPATH) $(TARGET_DIR) + cd $(INTERMEDIATE_DIR); $(COPY) `cat $(INTERMEDIATE_DIR)/package_filename`.sh $(TARGET_DIR) + +else + + @echo "========================= Performing Building RPM and DPKG packages" + sudo $(RMDIR) $(STAGING_DIR) + $(MKPATH) $(INTERMEDIATE_DIR) + python $(SCXPAL_DIR)/installer/InstallBuilder/installbuilder.py \ + --BASE_DIR=$(BASE_DIR) \ + --TARGET_DIR=$(INTERMEDIATE_DIR) \ + --INTERMEDIATE_DIR=$(INSTALLER_TMPDIR) \ + --STAGING_DIR=$(STAGING_DIR) \ + --BUILD_TYPE=$(BUILD_TYPE) \ + --BUILD_CONFIGURATION=$(BUILD_CONFIGURATION) \ + --PFARCH=$(PF_ARCH) \ + --PFDISTRO=$(PF_DISTRO) \ + --PFMAJOR=$(PF_MAJOR) \ + --PFMINOR=$(PF_MINOR) \ + --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ + --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ + --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ + $(DPKG_LOCATION) \ + --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ + --DATAFILE_PATH=$(BASE_DIR)/installer/datafiles \ + base_container.data linux.data linux_dpkg.data + +endif diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index ec0728c01..d41ba1960 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -37,7 +37,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_services.rb; source/code/plugin/in_kube_services.rb; 644; root; root /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root - +/opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root %Links /opt/omi/lib/libcontainer.${{SHLIB_EXT}}; /opt/microsoft/docker-cimprov/lib/libcontainer.${{SHLIB_EXT}}; 644; root; root @@ -76,6 +76,9 @@ MAINTAINER: 'Microsoft Corporation' /var/opt/microsoft/docker-cimprov/state/ImageInventory; 755; root; root /var/opt/microsoft/docker-cimprov/log; 755; root; root +/opt/td-agent-bit; 755; root; root;sysdir +/opt/td-agent-bit/bin; 755; root; root;sysdir + %Dependencies %Postinstall_10 diff --git a/source/code/go/src/plugins/Makefile b/source/code/go/src/plugins/Makefile new file mode 100644 index 000000000..dfdc65d81 --- /dev/null +++ b/source/code/go/src/plugins/Makefile @@ -0,0 +1,20 @@ +GITVERSION := 0.1 +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + BUILDDATE := $(shell date --rfc-3339=seconds) +endif +ifeq ($(UNAME_S),Darwin) + BUILDDATE := $(shell gdate --rfc-3339=seconds) +endif + +fbplugin: + go build -ldflags "-X 'main.revision=$(GITVERSION)' -X 'main.builddate=$(BUILDDATE)'" -buildmode=c-shared -o out_oms.so . + +test: + go test -cover -race -coverprofile=coverage.txt -covermode=atomic + +glide: + glide install + +clean: + rm -rf *.so *.h *~ diff --git a/source/code/go/src/plugins/glide.lock b/source/code/go/src/plugins/glide.lock new file mode 100644 index 000000000..79745820b --- /dev/null +++ b/source/code/go/src/plugins/glide.lock @@ -0,0 +1,209 @@ +hash: a4b073d827b5cbb4a772dada9ff3bcf55c55afc3cda83ddec1e6edcdca8e219a +updated: 2018-09-06T04:07:01.808678175Z +imports: +- name: github.com/fluent/fluent-bit-go + version: c4a158a6e3a793166c6ecfa2d5c80d71eada8959 + subpackages: + - output +- name: github.com/ghodss/yaml + version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee +- name: github.com/gogo/protobuf + version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 + subpackages: + - proto + - sortkeys +- name: github.com/golang/glog + version: 44145f04b68cf362d9c4df2182967c2275eaefed +- name: github.com/golang/protobuf + version: b4deda0973fb4c70b50d226b1af49f3da59f5265 + subpackages: + - proto + - ptypes + - ptypes/any + - ptypes/duration + - ptypes/timestamp +- name: github.com/google/btree + version: 7d79101e329e5a3adf994758c578dab82b90c017 +- name: github.com/google/gofuzz + version: 44d81051d367757e1c7c6a5a86423ece9afcf63c +- name: github.com/googleapis/gnostic + version: 0c5108395e2debce0d731cf0287ddf7242066aba + subpackages: + - OpenAPIv2 + - compiler + - extensions +- name: github.com/gregjones/httpcache + version: 787624de3eb7bd915c329cba748687a3b22666a6 + subpackages: + - diskcache +- name: github.com/json-iterator/go + version: f2b4162afba35581b6d4a50d3b8f34e33c144682 +- name: github.com/mitchellh/mapstructure + version: fa473d140ef3c6adf42d6b391fe76707f1f243c8 +- name: github.com/modern-go/concurrent + version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 +- name: github.com/modern-go/reflect2 + version: 05fbef0ca5da472bbf96c9322b84a53edc03c9fd +- name: github.com/peterbourgon/diskv + version: 5f041e8faa004a95c88a202771f4cc3e991971e6 +- name: github.com/ugorji/go + version: 00b869d2f4a5e27445c2d916fa106fc72c106d4c + subpackages: + - codec +- name: golang.org/x/crypto + version: 49796115aa4b964c318aad4f3084fdb41e9aa067 + subpackages: + - ssh/terminal +- name: golang.org/x/net + version: 1c05540f6879653db88113bc4a2b70aec4bd491f + subpackages: + - context + - html + - html/atom + - http2 + - http2/hpack + - idna + - lex/httplex + - websocket +- name: golang.org/x/sys + version: 95c6576299259db960f6c5b9b69ea52422860fce + subpackages: + - unix + - windows +- name: golang.org/x/text + version: b19bf474d317b857955b12035d2c5acb57ce8b01 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: golang.org/x/time + version: f51c12702a4d776e4c1fa9b0fabab841babae631 + subpackages: + - rate +- name: gopkg.in/inf.v0 + version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 +- name: gopkg.in/yaml.v2 + version: 670d4cfef0544295bc27a114dbac37980d83185a +- name: k8s.io/api + version: 072894a440bdee3a891dea811fe42902311cd2a3 + subpackages: + - admissionregistration/v1alpha1 + - admissionregistration/v1beta1 + - apps/v1 + - apps/v1beta1 + - apps/v1beta2 + - authentication/v1 + - authentication/v1beta1 + - authorization/v1 + - authorization/v1beta1 + - autoscaling/v1 + - autoscaling/v2beta1 + - batch/v1 + - batch/v1beta1 + - batch/v2alpha1 + - certificates/v1beta1 + - core/v1 + - events/v1beta1 + - extensions/v1beta1 + - imagepolicy/v1alpha1 + - networking/v1 + - policy/v1beta1 + - rbac/v1 + - rbac/v1alpha1 + - rbac/v1beta1 + - scheduling/v1alpha1 + - scheduling/v1beta1 + - settings/v1alpha1 + - storage/v1 + - storage/v1alpha1 + - storage/v1beta1 +- name: k8s.io/apimachinery + version: 103fd098999dc9c0c88536f5c9ad2e5da39373ae + subpackages: + - pkg/api/errors + - pkg/api/meta + - pkg/api/resource + - pkg/apis/meta/v1 + - pkg/apis/meta/v1/unstructured + - pkg/apis/meta/v1beta1 + - pkg/conversion + - pkg/conversion/queryparams + - pkg/fields + - pkg/labels + - pkg/runtime + - pkg/runtime/schema + - pkg/runtime/serializer + - pkg/runtime/serializer/json + - pkg/runtime/serializer/protobuf + - pkg/runtime/serializer/recognizer + - pkg/runtime/serializer/streaming + - pkg/runtime/serializer/versioning + - pkg/selection + - pkg/types + - pkg/util/clock + - pkg/util/errors + - pkg/util/framer + - pkg/util/intstr + - pkg/util/json + - pkg/util/net + - pkg/util/runtime + - pkg/util/sets + - pkg/util/validation + - pkg/util/validation/field + - pkg/util/wait + - pkg/util/yaml + - pkg/version + - pkg/watch + - third_party/forked/golang/reflect +- name: k8s.io/client-go + version: 7d04d0e2a0a1a4d4a1cd6baa432a2301492e4e65 + subpackages: + - discovery + - kubernetes + - kubernetes/scheme + - kubernetes/typed/admissionregistration/v1alpha1 + - kubernetes/typed/admissionregistration/v1beta1 + - kubernetes/typed/apps/v1 + - kubernetes/typed/apps/v1beta1 + - kubernetes/typed/apps/v1beta2 + - kubernetes/typed/authentication/v1 + - kubernetes/typed/authentication/v1beta1 + - kubernetes/typed/authorization/v1 + - kubernetes/typed/authorization/v1beta1 + - kubernetes/typed/autoscaling/v1 + - kubernetes/typed/autoscaling/v2beta1 + - kubernetes/typed/batch/v1 + - kubernetes/typed/batch/v1beta1 + - kubernetes/typed/batch/v2alpha1 + - kubernetes/typed/certificates/v1beta1 + - kubernetes/typed/core/v1 + - kubernetes/typed/events/v1beta1 + - kubernetes/typed/extensions/v1beta1 + - kubernetes/typed/networking/v1 + - kubernetes/typed/policy/v1beta1 + - kubernetes/typed/rbac/v1 + - kubernetes/typed/rbac/v1alpha1 + - kubernetes/typed/rbac/v1beta1 + - kubernetes/typed/scheduling/v1alpha1 + - kubernetes/typed/scheduling/v1beta1 + - kubernetes/typed/settings/v1alpha1 + - kubernetes/typed/storage/v1 + - kubernetes/typed/storage/v1alpha1 + - kubernetes/typed/storage/v1beta1 + - pkg/apis/clientauthentication + - pkg/apis/clientauthentication/v1alpha1 + - pkg/apis/clientauthentication/v1beta1 + - pkg/version + - plugin/pkg/client/auth/exec + - rest + - rest/watch + - tools/clientcmd/api + - tools/metrics + - tools/reference + - transport + - util/cert + - util/connrotation + - util/flowcontrol + - util/integer +testImports: [] diff --git a/source/code/go/src/plugins/glide.yaml b/source/code/go/src/plugins/glide.yaml new file mode 100644 index 000000000..b986ece21 --- /dev/null +++ b/source/code/go/src/plugins/glide.yaml @@ -0,0 +1,15 @@ +package: plugins +import: +- package: github.com/fluent/fluent-bit-go + subpackages: + - output +- package: github.com/mitchellh/mapstructure + version: ^1.0.0 +- package: k8s.io/apimachinery + subpackages: + - pkg/apis/meta/v1 +- package: k8s.io/client-go + version: ^8.0.0 + subpackages: + - kubernetes + - rest diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go new file mode 100644 index 000000000..f7e4289fe --- /dev/null +++ b/source/code/go/src/plugins/oms.go @@ -0,0 +1,263 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + // KeyFile is the path to the private key file used for auth + KeyFile = flag.String("key", "/shared/data/oms.key", "Private Key File") + // CertFile is the path to the cert used for auth + CertFile = flag.String("cert", "/shared/data/oms.crt", "OMS Agent Certificate") + // OMSEndpoint ingestion endpoint + OMSEndpoint string +) + +var ( + // ImageIDMap caches the container id to image mapping + ImageIDMap map[string]string + // NameIDMap caches the container it to Name mapping + NameIDMap map[string]string + // IgnoreIDSet set of container Ids of kube-system pods + IgnoreIDSet map[string]bool +) + +var ( + // FLBLogger stream + FLBLogger = createLogger() + // Log method + Log = FLBLogger.Printf +) + +const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" + +// ContainerInventory represents the container info +type ContainerInventory struct { + ElementName string `json:"ElementName"` + CreatedTime string `json:"CreatedTime"` + State string `json:"State"` + ExitCode int `json:"ExitCode"` + StartedTime string `json:"StartedTime"` + FinishedTime string `json:"FinishedTime"` + ImageID string `json:"ImageId"` + Image string `json:"Image"` + Repository string `json:"Repository"` + ImageTag string `json:"ImageTag"` + ComposeGroup string `json:"ComposeGroup"` + ContainerHostname string `json:"ContainerHostname"` + Computer string `json:"Computer"` + Command string `json:"Command"` + EnvironmentVar string `json:"EnvironmentVar"` + Ports string `json:"Ports"` + Links string `json:"Links"` +} + +// DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin +type DataItem struct { + LogEntry string `json:"LogEntry"` + LogEntrySource string `json:"LogEntrySource"` + LogEntryTimeStamp string `json:"LogEntryTimeStamp"` + ID string `json:"Id"` + Image string `json:"Image"` + Name string `json:"Name"` + SourceSystem string `json:"SourceSystem"` + Computer string `json:"Computer"` +} + +// ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point +type ContainerLogBlob struct { + DataType string `json:"DataType"` + IPName string `json:"IPName"` + DataItems []DataItem `json:"DataItems"` +} + +func populateMaps() { + files, err := ioutil.ReadDir(containerInventoryPath) + + if err != nil { + Log("error when reading container inventory") + log.Fatal(err.Error()) + } + + for _, file := range files { + fullPath := fmt.Sprintf("%s/%s", containerInventoryPath, file.Name()) + fileContent, err := ioutil.ReadFile(fullPath) + if err != nil { + Log("Error reading file content %s", fullPath) + log.Fatal(err) + } + var containerInventory ContainerInventory + unmarshallErr := json.Unmarshal(fileContent, &containerInventory) + + if unmarshallErr != nil { + Log("Unmarshall error when reading file %s", fullPath) + log.Fatal(unmarshallErr) + } + + ImageIDMap[file.Name()] = containerInventory.Image + NameIDMap[file.Name()] = containerInventory.ElementName + } +} + +func createLogger() *log.Logger { + logfile, err := os.Create(filepath.Join("/shared/data", fmt.Sprintf("fluent-bit-runtime_%s.log", time.Now().Format("2006-01-02T15-04-05")))) + if err != nil { + panic(err.Error()) + } + + return log.New(logfile, "", 0) +} + +func initMaps() { + ImageIDMap = make(map[string]string) + NameIDMap = make(map[string]string) + + populateMaps() + + for range time.Tick(time.Second * 60) { + populateMaps() + } +} + +func updateIgnoreContainerIds() { + IgnoreIDSet = make(map[string]bool) + + updateKubeSystemContainerIDs() + + for range time.Tick(time.Second * 300) { + updateKubeSystemContainerIDs() + } +} + +func updateKubeSystemContainerIDs() { + + if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { + Log("Kube System Log Collection is ENABLED.") + return + } + + Log("Kube System Log Collection is DISABLED. Collecting containerIds to drop their records") + config, err := rest.InClusterConfig() + if err != nil { + Log("Error getting config") + panic(err.Error()) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + Log("Error getting clientset") + panic(err.Error()) + } + + pods, err := clientset.CoreV1().Pods("kube-system").List(metav1.ListOptions{}) + if err != nil { + Log("Error getting pods") + panic(err.Error()) + } + + for _, pod := range pods.Items { + for _, status := range pod.Status.ContainerStatuses { + IgnoreIDSet[status.ContainerID[9:len(status.ContainerID)]] = true + } + } +} + +// PostDataHelper sends data to the OMS endpoint +func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) { + + var dataItems []DataItem + for _, record := range tailPluginRecords { + + id := toString(record["Id"]) + + // if Id is in the list of Ids to drop (Kube-system containers) continue + if containsKey(IgnoreIDSet, id) { + Log("Dropping record with id %s since it is a kube-system log entry and log collection is disabled for KubeSystem", id) + continue + } + + var dataItem DataItem + stringMap := make(map[string]string) + + // convert map[interface{}]interface{} to map[string]string + for key, value := range record { + strKey := fmt.Sprintf("%v", key) + strValue := toString(value) + stringMap[strKey] = strValue + } + + // TODO : dilipr check for the existence of the key and update the map + stringMap["Image"] = ImageIDMap[id] + stringMap["Name"] = NameIDMap[id] + mapstructure.Decode(stringMap, &dataItem) + dataItems = append(dataItems, dataItem) + } + + logEntry := ContainerLogBlob{ + DataType: "CONTAINER_LOG_BLOB", + IPName: "Containers", + DataItems: dataItems} + + marshalled, err := json.Marshal(logEntry) + + cert, err := tls.LoadX509KeyPair(*CertFile, *KeyFile) + if err != nil { + Log("Error when loading cert") + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + tlsConfig.BuildNameToCertificate() + transport := &http.Transport{TLSClientConfig: tlsConfig} + + client := &http.Client{Transport: transport} + req, _ := http.NewRequest("POST", OMSEndpoint, bytes.NewBuffer(marshalled)) + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + Log("Error when sending request") + } + + statusCode := resp.Status + Log("Status Code: %s", statusCode) +} + +func containsKey(currentMap map[string]bool, key string) bool { + _, c := currentMap[key] + return c +} + +func readConfig() { + workspaceIDFile := "/shared/data/workspaceId" + workspaceID, err := ioutil.ReadFile(workspaceIDFile) + if err != nil { + Log("Error when reading workspaceId file") + } + + OMSEndpoint = fmt.Sprintf("https://%s.ods.opinsights.azure.com/OperationalData.svc/PostJsonDataItems", strings.TrimSpace(string(workspaceID))) + Log("OMSEndpoint %s \n\n", OMSEndpoint) +} + +func toString(s interface{}) string { + value := s.([]uint8) + return string([]byte(value[:])) +} diff --git a/source/code/go/src/plugins/out_oms.go b/source/code/go/src/plugins/out_oms.go new file mode 100644 index 000000000..4ce24eee7 --- /dev/null +++ b/source/code/go/src/plugins/out_oms.go @@ -0,0 +1,65 @@ +package main + +import ( + "time" + + "github.com/fluent/fluent-bit-go/output" +) +import ( + "C" + "unsafe" +) + +//export FLBPluginRegister +func FLBPluginRegister(ctx unsafe.Pointer) int { + return output.FLBPluginRegister(ctx, "oms", "Stdout GO!") +} + +//export FLBPluginInit +// (fluentbit will call this) +// ctx (context) pointer to fluentbit context (state/ c code) +func FLBPluginInit(ctx unsafe.Pointer) int { + readConfig() + go initMaps() + go updateIgnoreContainerIds() + return output.FLB_OK +} + +//export FLBPluginFlush +func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { + var count int + var ret int + var record map[interface{}]interface{} + var records []map[interface{}]interface{} + + start := time.Now() + // Create Fluent Bit decoder + dec := output.NewDecoder(data, int(length)) + + // Iterate Records + count = 0 + for { + // Extract Record + ret, _, record = output.GetRecord(dec) + if ret != 0 { + break + } + records = append(records, record) + count++ + } + PostDataHelper(records) + + elapsed := time.Since(start) + + Log("Successfully flushed %d records in %s", len(records), elapsed) + + return output.FLB_OK +} + +//export FLBPluginExit +func FLBPluginExit() int { + return output.FLB_OK +} + +func main() { +} From abb728239a8575d4eabe97a089664fa2690f0286 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 7 Sep 2018 00:22:36 +0000 Subject: [PATCH 2/8] Adding fluent-bit-config file to base container --- installer/conf/td-agent-bit.conf | 53 +++++++++++++++++++++++++ installer/datafiles/base_container.data | 1 + 2 files changed, 54 insertions(+) create mode 100644 installer/conf/td-agent-bit.conf diff --git a/installer/conf/td-agent-bit.conf b/installer/conf/td-agent-bit.conf new file mode 100644 index 000000000..029444a42 --- /dev/null +++ b/installer/conf/td-agent-bit.conf @@ -0,0 +1,53 @@ +[SERVICE] + Flush 1 + Log_Level info + Daemon off + Parsers_File parsers.conf + Log_File /var/log/fluent-bit.log + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name tail + Tag oms.container.log.* + Path /var/log/containers/*.log + Exclude_Path *azureproxy*.log,*kube-dns*.log,*kubernetes-dashboard*.log,*heapster*.log,*addon-http-application*.log,*omsagent*.log,*tunnelfront*.log,*kube-svc*log + DB /var/log/fblogs.db + Parser docker + Mem_Buf_Limit 30m + +[FILTER] + Name kubernetes + Match oms.container.log.* + Kube_URL https://kubernetes.default.svc.cluster.local:443 + Merge_JSON_Log On + K8S-Logging.Parser On +[FILTER] + Name nest + Match oms.container.log.* + Operation lift + Nested_under kubernetes +[FILTER] + Name record_modifier + Match oms.container.log.* + Whitelist_key log + Whitelist_key stream + Whitelist_key time + Whitelist_key docker_id + Whitelist_key pod_name + Whitelist_key host +[FILTER] + Name modify + Match oms.container.log.* + Rename log LogEntry + Rename stream LogEntrySource + Rename time LogEntryTimeStamp + Rename docker_id Id + Rename pod_name Name + Add_if_not_present SourceSystem Containers + Rename host Computer + +[OUTPUT] + Name oms + Match oms.container.log.* diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index d41ba1960..d0b5e2beb 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -38,6 +38,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root +/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; installer/conf/td-agent-bit.conf; 644; root; root %Links /opt/omi/lib/libcontainer.${{SHLIB_EXT}}; /opt/microsoft/docker-cimprov/lib/libcontainer.${{SHLIB_EXT}}; 644; root; root From c9163d330016f94edcf89a25bb3e2508607067a6 Mon Sep 17 00:00:00 2001 From: r-dilip Date: Tue, 11 Sep 2018 15:53:21 -0700 Subject: [PATCH 3/8] PR Feedback --- installer/conf/out_oms.conf | 4 + installer/conf/td-agent-bit.conf | 33 +- source/code/go/src/plugins/oms.go | 609 +++++++++++++++----------- source/code/go/src/plugins/out_oms.go | 123 +++--- source/code/go/src/plugins/utils.go | 67 +++ 5 files changed, 483 insertions(+), 353 deletions(-) create mode 100644 installer/conf/out_oms.conf create mode 100644 source/code/go/src/plugins/utils.go diff --git a/installer/conf/out_oms.conf b/installer/conf/out_oms.conf new file mode 100644 index 000000000..c69436a1e --- /dev/null +++ b/installer/conf/out_oms.conf @@ -0,0 +1,4 @@ +omsadmin_conf_path=/etc/opt/microsoft/omsagent/conf/omsadmin.conf +cert_file_path=/etc/opt/microsoft/omsagent/certs/oms.crt +key_file_path=/etc/opt/microsoft/omsagent/certs/oms.key +container_host_file_path=/var/opt/microsoft/docker-cimprov/state/containerhostname \ No newline at end of file diff --git a/installer/conf/td-agent-bit.conf b/installer/conf/td-agent-bit.conf index 029444a42..e14161307 100644 --- a/installer/conf/td-agent-bit.conf +++ b/installer/conf/td-agent-bit.conf @@ -1,53 +1,36 @@ [SERVICE] - Flush 1 + Flush 5 Log_Level info - Daemon off - Parsers_File parsers.conf + Daemon on + Parsers_File /etc/td-agent-bit/parsers.conf Log_File /var/log/fluent-bit.log - HTTP_Server On - HTTP_Listen 0.0.0.0 - HTTP_Port 2020 [INPUT] Name tail Tag oms.container.log.* Path /var/log/containers/*.log - Exclude_Path *azureproxy*.log,*kube-dns*.log,*kubernetes-dashboard*.log,*heapster*.log,*addon-http-application*.log,*omsagent*.log,*tunnelfront*.log,*kube-svc*log DB /var/log/fblogs.db Parser docker Mem_Buf_Limit 30m + Path_Key filepath -[FILTER] - Name kubernetes - Match oms.container.log.* - Kube_URL https://kubernetes.default.svc.cluster.local:443 - Merge_JSON_Log On - K8S-Logging.Parser On -[FILTER] - Name nest - Match oms.container.log.* - Operation lift - Nested_under kubernetes [FILTER] Name record_modifier Match oms.container.log.* Whitelist_key log Whitelist_key stream Whitelist_key time - Whitelist_key docker_id - Whitelist_key pod_name - Whitelist_key host + Whitelist_key filepath + [FILTER] Name modify Match oms.container.log.* Rename log LogEntry Rename stream LogEntrySource Rename time LogEntryTimeStamp - Rename docker_id Id - Rename pod_name Name + Rename filepath Filepath Add_if_not_present SourceSystem Containers - Rename host Computer [OUTPUT] Name oms - Match oms.container.log.* + Match oms.container.log.* \ No newline at end of file diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index f7e4289fe..8900e4ac4 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -1,263 +1,346 @@ -package main - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/mitchellh/mapstructure" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -var ( - // KeyFile is the path to the private key file used for auth - KeyFile = flag.String("key", "/shared/data/oms.key", "Private Key File") - // CertFile is the path to the cert used for auth - CertFile = flag.String("cert", "/shared/data/oms.crt", "OMS Agent Certificate") - // OMSEndpoint ingestion endpoint - OMSEndpoint string -) - -var ( - // ImageIDMap caches the container id to image mapping - ImageIDMap map[string]string - // NameIDMap caches the container it to Name mapping - NameIDMap map[string]string - // IgnoreIDSet set of container Ids of kube-system pods - IgnoreIDSet map[string]bool -) - -var ( - // FLBLogger stream - FLBLogger = createLogger() - // Log method - Log = FLBLogger.Printf -) - -const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" - -// ContainerInventory represents the container info -type ContainerInventory struct { - ElementName string `json:"ElementName"` - CreatedTime string `json:"CreatedTime"` - State string `json:"State"` - ExitCode int `json:"ExitCode"` - StartedTime string `json:"StartedTime"` - FinishedTime string `json:"FinishedTime"` - ImageID string `json:"ImageId"` - Image string `json:"Image"` - Repository string `json:"Repository"` - ImageTag string `json:"ImageTag"` - ComposeGroup string `json:"ComposeGroup"` - ContainerHostname string `json:"ContainerHostname"` - Computer string `json:"Computer"` - Command string `json:"Command"` - EnvironmentVar string `json:"EnvironmentVar"` - Ports string `json:"Ports"` - Links string `json:"Links"` -} - -// DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin -type DataItem struct { - LogEntry string `json:"LogEntry"` - LogEntrySource string `json:"LogEntrySource"` - LogEntryTimeStamp string `json:"LogEntryTimeStamp"` - ID string `json:"Id"` - Image string `json:"Image"` - Name string `json:"Name"` - SourceSystem string `json:"SourceSystem"` - Computer string `json:"Computer"` -} - -// ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point -type ContainerLogBlob struct { - DataType string `json:"DataType"` - IPName string `json:"IPName"` - DataItems []DataItem `json:"DataItems"` -} - -func populateMaps() { - files, err := ioutil.ReadDir(containerInventoryPath) - - if err != nil { - Log("error when reading container inventory") - log.Fatal(err.Error()) - } - - for _, file := range files { - fullPath := fmt.Sprintf("%s/%s", containerInventoryPath, file.Name()) - fileContent, err := ioutil.ReadFile(fullPath) - if err != nil { - Log("Error reading file content %s", fullPath) - log.Fatal(err) - } - var containerInventory ContainerInventory - unmarshallErr := json.Unmarshal(fileContent, &containerInventory) - - if unmarshallErr != nil { - Log("Unmarshall error when reading file %s", fullPath) - log.Fatal(unmarshallErr) - } - - ImageIDMap[file.Name()] = containerInventory.Image - NameIDMap[file.Name()] = containerInventory.ElementName - } -} - -func createLogger() *log.Logger { - logfile, err := os.Create(filepath.Join("/shared/data", fmt.Sprintf("fluent-bit-runtime_%s.log", time.Now().Format("2006-01-02T15-04-05")))) - if err != nil { - panic(err.Error()) - } - - return log.New(logfile, "", 0) -} - -func initMaps() { - ImageIDMap = make(map[string]string) - NameIDMap = make(map[string]string) - - populateMaps() - - for range time.Tick(time.Second * 60) { - populateMaps() - } -} - -func updateIgnoreContainerIds() { - IgnoreIDSet = make(map[string]bool) - - updateKubeSystemContainerIDs() - - for range time.Tick(time.Second * 300) { - updateKubeSystemContainerIDs() - } -} - -func updateKubeSystemContainerIDs() { - - if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { - Log("Kube System Log Collection is ENABLED.") - return - } - - Log("Kube System Log Collection is DISABLED. Collecting containerIds to drop their records") - config, err := rest.InClusterConfig() - if err != nil { - Log("Error getting config") - panic(err.Error()) - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - Log("Error getting clientset") - panic(err.Error()) - } - - pods, err := clientset.CoreV1().Pods("kube-system").List(metav1.ListOptions{}) - if err != nil { - Log("Error getting pods") - panic(err.Error()) - } - - for _, pod := range pods.Items { - for _, status := range pod.Status.ContainerStatuses { - IgnoreIDSet[status.ContainerID[9:len(status.ContainerID)]] = true - } - } -} - -// PostDataHelper sends data to the OMS endpoint -func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) { - - var dataItems []DataItem - for _, record := range tailPluginRecords { - - id := toString(record["Id"]) - - // if Id is in the list of Ids to drop (Kube-system containers) continue - if containsKey(IgnoreIDSet, id) { - Log("Dropping record with id %s since it is a kube-system log entry and log collection is disabled for KubeSystem", id) - continue - } - - var dataItem DataItem - stringMap := make(map[string]string) - - // convert map[interface{}]interface{} to map[string]string - for key, value := range record { - strKey := fmt.Sprintf("%v", key) - strValue := toString(value) - stringMap[strKey] = strValue - } - - // TODO : dilipr check for the existence of the key and update the map - stringMap["Image"] = ImageIDMap[id] - stringMap["Name"] = NameIDMap[id] - mapstructure.Decode(stringMap, &dataItem) - dataItems = append(dataItems, dataItem) - } - - logEntry := ContainerLogBlob{ - DataType: "CONTAINER_LOG_BLOB", - IPName: "Containers", - DataItems: dataItems} - - marshalled, err := json.Marshal(logEntry) - - cert, err := tls.LoadX509KeyPair(*CertFile, *KeyFile) - if err != nil { - Log("Error when loading cert") - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - } - - tlsConfig.BuildNameToCertificate() - transport := &http.Transport{TLSClientConfig: tlsConfig} - - client := &http.Client{Transport: transport} - req, _ := http.NewRequest("POST", OMSEndpoint, bytes.NewBuffer(marshalled)) - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - if err != nil { - Log("Error when sending request") - } - - statusCode := resp.Status - Log("Status Code: %s", statusCode) -} - -func containsKey(currentMap map[string]bool, key string) bool { - _, c := currentMap[key] - return c -} - -func readConfig() { - workspaceIDFile := "/shared/data/workspaceId" - workspaceID, err := ioutil.ReadFile(workspaceIDFile) - if err != nil { - Log("Error when reading workspaceId file") - } - - OMSEndpoint = fmt.Sprintf("https://%s.ods.opinsights.azure.com/OperationalData.svc/PostJsonDataItems", strings.TrimSpace(string(workspaceID))) - Log("OMSEndpoint %s \n\n", OMSEndpoint) -} - -func toString(s interface{}) string { - value := s.([]uint8) - return string([]byte(value[:])) -} +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "sync" + "time" + + "github.com/fluent/fluent-bit-go/output" + "github.com/mitchellh/mapstructure" + lumberjack "gopkg.in/natefinch/lumberjack.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// DataType for Container Log +const DataType = "CONTAINER_LOG_BLOB" + +// IPName for Container Log +const IPName = "Containers" + +var ( + // PluginConfiguration the plugins configuration + PluginConfiguration map[string]string + // HTTPClient for making POST requests to OMSEndpoint + HTTPClient http.Client + // OMSEndpoint ingestion endpoint + OMSEndpoint string + // Computer (Hostname) when ingesting into ContainerLog table + Computer string +) + +var ( + // ImageIDMap caches the container id to image mapping + ImageIDMap map[string]string + // NameIDMap caches the container it to Name mapping + NameIDMap map[string]string + // IgnoreIDSet set of container Ids of kube-system pods + IgnoreIDSet map[string]bool +) + +var ( + // FLBLogger stream + FLBLogger = createLogger() + + // Log wrapper function + Log = FLBLogger.Printf +) + +// Log wrapper to Log with timestamp +// func Log(format string, v ...interface{}) { +// FLBLogger.SetPrefix(time.Now().Format("2006-01-02 15:04:05") + " : ") +// FLBLogger.Printf(format, v) +// } + +const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" + +// ContainerInventory represents the container info +type ContainerInventory struct { + ElementName string `json:"ElementName"` + CreatedTime string `json:"CreatedTime"` + State string `json:"State"` + ExitCode int `json:"ExitCode"` + StartedTime string `json:"StartedTime"` + FinishedTime string `json:"FinishedTime"` + ImageID string `json:"ImageId"` + Image string `json:"Image"` + Repository string `json:"Repository"` + ImageTag string `json:"ImageTag"` + ComposeGroup string `json:"ComposeGroup"` + ContainerHostname string `json:"ContainerHostname"` + Computer string `json:"Computer"` + Command string `json:"Command"` + EnvironmentVar string `json:"EnvironmentVar"` + Ports string `json:"Ports"` + Links string `json:"Links"` +} + +// DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin +type DataItem struct { + LogEntry string `json:"LogEntry"` + LogEntrySource string `json:"LogEntrySource"` + LogEntryTimeStamp string `json:"LogEntryTimeStamp"` + ContainerID string `json:"ContainerId"` + Image string `json:"Image"` + Name string `json:"Name"` + SourceSystem string `json:"SourceSystem"` + Computer string `json:"Computer"` + Filepath string `json:"Filepath"` +} + +// ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point +type ContainerLogBlob struct { + DataType string `json:"DataType"` + IPName string `json:"IPName"` + DataItems []DataItem `json:"DataItems"` +} + +func populateMaps() { + + Log("Updating ImageIDMap and NameIDMap") + var mutex = &sync.Mutex{} + files, err := ioutil.ReadDir(containerInventoryPath) + + if err != nil { + Log("error when reading container inventory %s\n", err.Error()) + } + + mutex.Lock() + ImageIDMap = make(map[string]string) + NameIDMap = make(map[string]string) + + for _, file := range files { + fullPath := fmt.Sprintf("%s/%s", containerInventoryPath, file.Name()) + fileContent, err := ioutil.ReadFile(fullPath) + if err != nil { + Log("Error reading file content %s", fullPath) + Log(err.Error()) + } + var containerInventory ContainerInventory + unmarshallErr := json.Unmarshal(fileContent, &containerInventory) + + if unmarshallErr != nil { + Log("Unmarshall error when reading file %s %s \n", fullPath, unmarshallErr.Error()) + } + + ImageIDMap[file.Name()] = containerInventory.Image + NameIDMap[file.Name()] = containerInventory.ElementName + } + mutex.Unlock() +} + +func createLogger() *log.Logger { + + var logfile *os.File + path := "/var/log/fluent-bit-out-oms-runtime.log" + if _, err := os.Stat(path); err == nil { + fmt.Printf("File Exists. Opening file in append mode...\n") + logfile, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + fmt.Printf(err.Error()) + } + } + + if _, err := os.Stat(path); os.IsNotExist(err) { + fmt.Printf("File Doesnt Exist. Creating file...\n") + logfile, err = os.Create(path) + if err != nil { + fmt.Printf(err.Error()) + } + } + + logger := log.New(logfile, "", 0) + + logger.SetOutput(&lumberjack.Logger{ + Filename: path, + MaxSize: 10, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, // false by default + }) + + logger.SetFlags(log.Ltime | log.Lshortfile | log.LstdFlags) + return logger +} + +func initMaps() { + ImageIDMap = make(map[string]string) + NameIDMap = make(map[string]string) + + populateMaps() + + for range time.Tick(time.Second * 60) { + populateMaps() + } +} + +func updateIgnoreContainerIds() { + IgnoreIDSet = make(map[string]bool) + + updateKubeSystemContainerIDs() + + for range time.Tick(time.Second * 300) { + updateKubeSystemContainerIDs() + } +} + +func updateKubeSystemContainerIDs() { + + var mutex = &sync.Mutex{} + mutex.Lock() + IgnoreIDSet = make(map[string]bool) + + if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { + Log("Kube System Log Collection is ENABLED.") + return + } + + Log("Kube System Log Collection is DISABLED. Collecting containerIds to drop their records") + config, err := rest.InClusterConfig() + if err != nil { + Log("Error getting config %s\n", err.Error()) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + Log("Error getting clientset %s", err.Error()) + } + + pods, err := clientset.CoreV1().Pods("kube-system").List(metav1.ListOptions{}) + if err != nil { + Log("Error getting pods %s\n", err.Error()) + } + + for _, pod := range pods.Items { + for _, status := range pod.Status.ContainerStatuses { + IgnoreIDSet[status.ContainerID[9:len(status.ContainerID)]] = true + } + } + mutex.Unlock() +} + +// PostDataHelper sends data to the OMS endpoint +func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { + + start := time.Now() + var dataItems []DataItem + for _, record := range tailPluginRecords { + + //id := toString(record["Id"]) + containerID := getContainerIDFromFilePath(toString(record["Filepath"])) + + // if Id is in the list of Ids to drop (Kube-system containers) continue + if containsKey(IgnoreIDSet, containerID) { + continue + } + + var dataItem DataItem + stringMap := make(map[string]string) + + // convert map[interface{}]interface{} to map[string]string + for key, value := range record { + strKey := fmt.Sprintf("%v", key) + strValue := toString(value) + stringMap[strKey] = strValue + } + + stringMap["Image"] = ImageIDMap[containerID] + stringMap["Name"] = NameIDMap[containerID] + stringMap["Computer"] = Computer + mapstructure.Decode(stringMap, &dataItem) + dataItems = append(dataItems, dataItem) + } + + if len(dataItems) > 0 { + logEntry := ContainerLogBlob{ + DataType: DataType, + IPName: IPName, + DataItems: dataItems} + + marshalled, err := json.Marshal(logEntry) + req, _ := http.NewRequest("POST", OMSEndpoint, bytes.NewBuffer(marshalled)) + req.Header.Set("Content-Type", "application/json") + + resp, err := HTTPClient.Do(req) + if err != nil { + Log("Error when sending request %s \n", err.Error()) + } + + if resp == nil || resp.StatusCode != 200 { + return output.FLB_RETRY + } + + elapsed := time.Since(start) + Log("Successfully flushed %d records in %s", len(dataItems), elapsed) + } + + return output.FLB_OK +} + +func containsKey(currentMap map[string]bool, key string) bool { + _, c := currentMap[key] + return c +} + +func readConfig() { + workspaceIDFile := "/shared/data/workspaceId" + workspaceID, err := ioutil.ReadFile(workspaceIDFile) + if err != nil { + Log("Error when reading workspaceId file") + } + + OMSEndpoint = fmt.Sprintf("https://%s.ods.opinsights.azure.com/OperationalData.svc/PostJsonDataItems", strings.TrimSpace(string(workspaceID))) + Log("OMSEndpoint %s \n\n", OMSEndpoint) + + containerHostNameFile := "/var/opt/microsoft/docker-cimprov/state/containerhostname" + containerHostName, err := ioutil.ReadFile(containerHostNameFile) + if err != nil { + Log("Error when reading containerHostName file") + } + + Computer = toString(containerHostName) + Log("Computer == %s \n\n", Computer) +} + +func toString(s interface{}) string { + value := s.([]uint8) + return string([]byte(value[:])) +} + +func getContainerIDFromFilePath(filepath string) string { + start := strings.LastIndex(filepath, "-") + end := strings.LastIndex(filepath, ".") + return filepath[start+1 : end] +} + +// ReadConfig reads and populates plugin configuration +func ReadConfig(pluginConfPath string) map[string]string { + + pluginConf, err := ReadConfiguration(pluginConfPath) + omsadminConf, err := ReadConfiguration(pluginConf["omsadmin_conf_path"]) + + if err != nil { + Log(err.Error()) + } + + containerHostName, err := ioutil.ReadFile(pluginConf["container_host_file_path"]) + if err != nil { + Log("Error when reading containerHostName file %s", err.Error()) + } + + Computer = toString(containerHostName) + Log("Computer == %s \n", Computer) + + OMSEndpoint = omsadminConf["OMS_ENDPOINT"] + Log("OMSEndpoint %s", OMSEndpoint) + + return pluginConf +} diff --git a/source/code/go/src/plugins/out_oms.go b/source/code/go/src/plugins/out_oms.go index 4ce24eee7..f15022ae2 100644 --- a/source/code/go/src/plugins/out_oms.go +++ b/source/code/go/src/plugins/out_oms.go @@ -1,65 +1,58 @@ -package main - -import ( - "time" - - "github.com/fluent/fluent-bit-go/output" -) -import ( - "C" - "unsafe" -) - -//export FLBPluginRegister -func FLBPluginRegister(ctx unsafe.Pointer) int { - return output.FLBPluginRegister(ctx, "oms", "Stdout GO!") -} - -//export FLBPluginInit -// (fluentbit will call this) -// ctx (context) pointer to fluentbit context (state/ c code) -func FLBPluginInit(ctx unsafe.Pointer) int { - readConfig() - go initMaps() - go updateIgnoreContainerIds() - return output.FLB_OK -} - -//export FLBPluginFlush -func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { - var count int - var ret int - var record map[interface{}]interface{} - var records []map[interface{}]interface{} - - start := time.Now() - // Create Fluent Bit decoder - dec := output.NewDecoder(data, int(length)) - - // Iterate Records - count = 0 - for { - // Extract Record - ret, _, record = output.GetRecord(dec) - if ret != 0 { - break - } - records = append(records, record) - count++ - } - PostDataHelper(records) - - elapsed := time.Since(start) - - Log("Successfully flushed %d records in %s", len(records), elapsed) - - return output.FLB_OK -} - -//export FLBPluginExit -func FLBPluginExit() int { - return output.FLB_OK -} - -func main() { -} +package main + +import ( + "github.com/fluent/fluent-bit-go/output" +) +import ( + "C" + "unsafe" +) + +//export FLBPluginRegister +func FLBPluginRegister(ctx unsafe.Pointer) int { + return output.FLBPluginRegister(ctx, "oms", "Stdout GO!") +} + +//export FLBPluginInit +// (fluentbit will call this) +// ctx (context) pointer to fluentbit context (state/ c code) +func FLBPluginInit(ctx unsafe.Pointer) int { + Log("Initializing out_oms go plugin for fluentbit") + PluginConfiguration = ReadConfig("/etc/opt/microsoft/docker-cimprov/out_oms.conf") + CreateHTTPClient() + go initMaps() + go updateIgnoreContainerIds() + return output.FLB_OK +} + +//export FLBPluginFlush +func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { + var count int + var ret int + var record map[interface{}]interface{} + var records []map[interface{}]interface{} + + // Create Fluent Bit decoder + dec := output.NewDecoder(data, int(length)) + + // Iterate Records + count = 0 + for { + // Extract Record + ret, _, record = output.GetRecord(dec) + if ret != 0 { + break + } + records = append(records, record) + count++ + } + return PostDataHelper(records) +} + +//export FLBPluginExit +func FLBPluginExit() int { + return output.FLB_OK +} + +func main() { +} diff --git a/source/code/go/src/plugins/utils.go b/source/code/go/src/plugins/utils.go new file mode 100644 index 000000000..0e33f43f9 --- /dev/null +++ b/source/code/go/src/plugins/utils.go @@ -0,0 +1,67 @@ +package main + +import ( + "bufio" + "crypto/tls" + "log" + "net/http" + "os" + "strings" +) + +// ReadConfiguration reads a property file +func ReadConfiguration(filename string) (map[string]string, error) { + config := map[string]string{} + + if len(filename) == 0 { + return config, nil + } + + file, err := os.Open(filename) + if err != nil { + log.Fatal(err) + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + currentLine := scanner.Text() + if equalIndex := strings.Index(currentLine, "="); equalIndex >= 0 { + if key := strings.TrimSpace(currentLine[:equalIndex]); len(key) > 0 { + value := "" + if len(currentLine) > equalIndex { + value = strings.TrimSpace(currentLine[equalIndex+1:]) + } + config[key] = value + } + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + return nil, err + } + + return config, nil +} + +// CreateHTTPClient used to create the client for sending post requests to OMSEndpoint +func CreateHTTPClient() { + + cert, err := tls.LoadX509KeyPair(PluginConfiguration["cert_file_path"], PluginConfiguration["key_file_path"]) + if err != nil { + Log("Error when loading cert %s", err.Error()) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + tlsConfig.BuildNameToCertificate() + transport := &http.Transport{TLSClientConfig: tlsConfig} + + HTTPClient = http.Client{Transport: transport} + + Log("Successfully created HTTP Client") +} From 029a03853b780445760e82caf523ead1f80bccb2 Mon Sep 17 00:00:00 2001 From: r-dilip Date: Tue, 11 Sep 2018 21:32:59 -0700 Subject: [PATCH 4/8] Adding out_oms.conf to base_container.data --- installer/conf/td-agent-bit.conf | 69 ++++++++++++------------- installer/datafiles/base_container.data | 3 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/installer/conf/td-agent-bit.conf b/installer/conf/td-agent-bit.conf index e14161307..cf490c077 100644 --- a/installer/conf/td-agent-bit.conf +++ b/installer/conf/td-agent-bit.conf @@ -1,36 +1,35 @@ -[SERVICE] - Flush 5 - Log_Level info - Daemon on - Parsers_File /etc/td-agent-bit/parsers.conf - Log_File /var/log/fluent-bit.log - -[INPUT] - Name tail - Tag oms.container.log.* - Path /var/log/containers/*.log - DB /var/log/fblogs.db - Parser docker - Mem_Buf_Limit 30m - Path_Key filepath - -[FILTER] - Name record_modifier - Match oms.container.log.* - Whitelist_key log - Whitelist_key stream - Whitelist_key time - Whitelist_key filepath - -[FILTER] - Name modify - Match oms.container.log.* - Rename log LogEntry - Rename stream LogEntrySource - Rename time LogEntryTimeStamp - Rename filepath Filepath - Add_if_not_present SourceSystem Containers - -[OUTPUT] - Name oms +[SERVICE] + Flush 5 + Log_Level info + Parsers_File /etc/td-agent-bit/parsers.conf + Log_File /var/log/fluent-bit.log + +[INPUT] + Name tail + Tag oms.container.log.* + Path /var/log/containers/*.log + DB /var/log/fblogs.db + Parser docker + Mem_Buf_Limit 30m + Path_Key filepath + +[FILTER] + Name record_modifier + Match oms.container.log.* + Whitelist_key log + Whitelist_key stream + Whitelist_key time + Whitelist_key filepath + +[FILTER] + Name modify + Match oms.container.log.* + Rename log LogEntry + Rename stream LogEntrySource + Rename time LogEntryTimeStamp + Rename filepath Filepath + Add_if_not_present SourceSystem Containers + +[OUTPUT] + Name oms Match oms.container.log.* \ No newline at end of file diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index d0b5e2beb..85a128b2a 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -38,7 +38,8 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root -/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; installer/conf/td-agent-bit.conf; 644; root; root +/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; installer/conf/td-agent-bit.conf; 644; root; root +/etc/opt/microsoft/docker-cimprov/out_oms.conf; installer/conf/out_oms.conf; 644; root; root %Links /opt/omi/lib/libcontainer.${{SHLIB_EXT}}; /opt/microsoft/docker-cimprov/lib/libcontainer.${{SHLIB_EXT}}; 644; root; root From ccbd6689b5935e9c3efb7406c3d1210fdfcb820c Mon Sep 17 00:00:00 2001 From: r-dilip Date: Wed, 12 Sep 2018 17:35:21 -0700 Subject: [PATCH 5/8] PR Feedback --- source/code/go/src/plugins/oms.go | 63 +++++++++------------------ source/code/go/src/plugins/out_oms.go | 2 +- 2 files changed, 22 insertions(+), 43 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 8900e4ac4..25e6b5382 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -26,6 +26,8 @@ const DataType = "CONTAINER_LOG_BLOB" // IPName for Container Log const IPName = "Containers" +const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" + var ( // PluginConfiguration the plugins configuration PluginConfiguration map[string]string @@ -44,6 +46,9 @@ var ( NameIDMap map[string]string // IgnoreIDSet set of container Ids of kube-system pods IgnoreIDSet map[string]bool + + // DataUpdateMutex read and write mutex access to the container id set + DataUpdateMutex = &sync.Mutex{} ) var ( @@ -54,14 +59,6 @@ var ( Log = FLBLogger.Printf ) -// Log wrapper to Log with timestamp -// func Log(format string, v ...interface{}) { -// FLBLogger.SetPrefix(time.Now().Format("2006-01-02 15:04:05") + " : ") -// FLBLogger.Printf(format, v) -// } - -const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" - // ContainerInventory represents the container info type ContainerInventory struct { ElementName string `json:"ElementName"` @@ -106,17 +103,16 @@ type ContainerLogBlob struct { func populateMaps() { Log("Updating ImageIDMap and NameIDMap") - var mutex = &sync.Mutex{} + + DataUpdateMutex.Lock() + ImageIDMap = make(map[string]string) + NameIDMap = make(map[string]string) files, err := ioutil.ReadDir(containerInventoryPath) if err != nil { Log("error when reading container inventory %s\n", err.Error()) } - mutex.Lock() - ImageIDMap = make(map[string]string) - NameIDMap = make(map[string]string) - for _, file := range files { fullPath := fmt.Sprintf("%s/%s", containerInventoryPath, file.Name()) fileContent, err := ioutil.ReadFile(fullPath) @@ -134,13 +130,13 @@ func populateMaps() { ImageIDMap[file.Name()] = containerInventory.Image NameIDMap[file.Name()] = containerInventory.ElementName } - mutex.Unlock() + DataUpdateMutex.Unlock() } func createLogger() *log.Logger { var logfile *os.File - path := "/var/log/fluent-bit-out-oms-runtime.log" + path := "/var/opt/microsoft/docker-cimprov/log/fluent-bit-out-oms-runtime.log" if _, err := os.Stat(path); err == nil { fmt.Printf("File Exists. Opening file in append mode...\n") logfile, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) @@ -194,8 +190,7 @@ func updateIgnoreContainerIds() { func updateKubeSystemContainerIDs() { - var mutex = &sync.Mutex{} - mutex.Lock() + DataUpdateMutex.Lock() IgnoreIDSet = make(map[string]bool) if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { @@ -221,10 +216,11 @@ func updateKubeSystemContainerIDs() { for _, pod := range pods.Items { for _, status := range pod.Status.ContainerStatuses { - IgnoreIDSet[status.ContainerID[9:len(status.ContainerID)]] = true + lastSlashIndex := strings.LastIndex(status.ContainerID, "/") + IgnoreIDSet[status.ContainerID[lastSlashIndex+1:len(status.ContainerID)]] = true } } - mutex.Unlock() + DataUpdateMutex.Unlock() } // PostDataHelper sends data to the OMS endpoint @@ -232,12 +228,12 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { start := time.Now() var dataItems []DataItem + DataUpdateMutex.Lock() + for _, record := range tailPluginRecords { - //id := toString(record["Id"]) containerID := getContainerIDFromFilePath(toString(record["Filepath"])) - // if Id is in the list of Ids to drop (Kube-system containers) continue if containsKey(IgnoreIDSet, containerID) { continue } @@ -258,6 +254,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { mapstructure.Decode(stringMap, &dataItem) dataItems = append(dataItems, dataItem) } + DataUpdateMutex.Unlock() if len(dataItems) > 0 { logEntry := ContainerLogBlob{ @@ -270,15 +267,17 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { req.Header.Set("Content-Type", "application/json") resp, err := HTTPClient.Do(req) + elapsed := time.Since(start) + if err != nil { Log("Error when sending request %s \n", err.Error()) + Log("Failed to flush %d records after %s", len(dataItems), elapsed) } if resp == nil || resp.StatusCode != 200 { return output.FLB_RETRY } - elapsed := time.Since(start) Log("Successfully flushed %d records in %s", len(dataItems), elapsed) } @@ -290,26 +289,6 @@ func containsKey(currentMap map[string]bool, key string) bool { return c } -func readConfig() { - workspaceIDFile := "/shared/data/workspaceId" - workspaceID, err := ioutil.ReadFile(workspaceIDFile) - if err != nil { - Log("Error when reading workspaceId file") - } - - OMSEndpoint = fmt.Sprintf("https://%s.ods.opinsights.azure.com/OperationalData.svc/PostJsonDataItems", strings.TrimSpace(string(workspaceID))) - Log("OMSEndpoint %s \n\n", OMSEndpoint) - - containerHostNameFile := "/var/opt/microsoft/docker-cimprov/state/containerhostname" - containerHostName, err := ioutil.ReadFile(containerHostNameFile) - if err != nil { - Log("Error when reading containerHostName file") - } - - Computer = toString(containerHostName) - Log("Computer == %s \n\n", Computer) -} - func toString(s interface{}) string { value := s.([]uint8) return string([]byte(value[:])) diff --git a/source/code/go/src/plugins/out_oms.go b/source/code/go/src/plugins/out_oms.go index f15022ae2..ccbb37b2a 100644 --- a/source/code/go/src/plugins/out_oms.go +++ b/source/code/go/src/plugins/out_oms.go @@ -49,7 +49,7 @@ func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { return PostDataHelper(records) } -//export FLBPluginExit +// FLBPluginExit exits the plugin func FLBPluginExit() int { return output.FLB_OK } From 75bd22ac4507b81dbd362ff89e615a93ec881706 Mon Sep 17 00:00:00 2001 From: r-dilip Date: Thu, 13 Sep 2018 12:58:14 -0700 Subject: [PATCH 6/8] Making the critical section as small as possible --- source/code/go/src/plugins/oms.go | 32 ++++++++++++++++++--------- source/code/go/src/plugins/out_oms.go | 6 ++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 25e6b5382..8a3a79dc7 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -25,7 +25,6 @@ const DataType = "CONTAINER_LOG_BLOB" // IPName for Container Log const IPName = "Containers" - const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" var ( @@ -104,9 +103,9 @@ func populateMaps() { Log("Updating ImageIDMap and NameIDMap") - DataUpdateMutex.Lock() - ImageIDMap = make(map[string]string) - NameIDMap = make(map[string]string) + _imageIDMap := make(map[string]string) + _nameIDMap := make(map[string]string) + files, err := ioutil.ReadDir(containerInventoryPath) if err != nil { @@ -127,10 +126,20 @@ func populateMaps() { Log("Unmarshall error when reading file %s %s \n", fullPath, unmarshallErr.Error()) } - ImageIDMap[file.Name()] = containerInventory.Image - NameIDMap[file.Name()] = containerInventory.ElementName + _imageIDMap[file.Name()] = containerInventory.Image + _nameIDMap[file.Name()] = containerInventory.ElementName } + Log("Locking to update image and name maps") + DataUpdateMutex.Lock() + ImageIDMap = _imageIDMap + NameIDMap = _nameIDMap DataUpdateMutex.Unlock() + Log("Unlocking after updating image and name maps") + + for k, v := range ImageIDMap { + Log("ID ==> %s | Image ==> %s \n", k, v) + Log("ID ==> %s | Image ==> %s \n", k, NameIDMap[k]) + } } func createLogger() *log.Logger { @@ -190,9 +199,6 @@ func updateIgnoreContainerIds() { func updateKubeSystemContainerIDs() { - DataUpdateMutex.Lock() - IgnoreIDSet = make(map[string]bool) - if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { Log("Kube System Log Collection is ENABLED.") return @@ -214,13 +220,19 @@ func updateKubeSystemContainerIDs() { Log("Error getting pods %s\n", err.Error()) } + _ignoreIDSet := make(map[string]bool) for _, pod := range pods.Items { for _, status := range pod.Status.ContainerStatuses { lastSlashIndex := strings.LastIndex(status.ContainerID, "/") - IgnoreIDSet[status.ContainerID[lastSlashIndex+1:len(status.ContainerID)]] = true + _ignoreIDSet[status.ContainerID[lastSlashIndex+1:len(status.ContainerID)]] = true } } + + Log("Locking to update kube-system container IDs") + DataUpdateMutex.Lock() + IgnoreIDSet = _ignoreIDSet DataUpdateMutex.Unlock() + Log("Unlocking after updating kube-system container IDs") } // PostDataHelper sends data to the OMS endpoint diff --git a/source/code/go/src/plugins/out_oms.go b/source/code/go/src/plugins/out_oms.go index ccbb37b2a..6d064cb85 100644 --- a/source/code/go/src/plugins/out_oms.go +++ b/source/code/go/src/plugins/out_oms.go @@ -8,12 +8,12 @@ import ( "unsafe" ) -//export FLBPluginRegister +// FLBPluginRegister registers the plugin func FLBPluginRegister(ctx unsafe.Pointer) int { return output.FLBPluginRegister(ctx, "oms", "Stdout GO!") } -//export FLBPluginInit +// FLBPluginInit initializes the plugin // (fluentbit will call this) // ctx (context) pointer to fluentbit context (state/ c code) func FLBPluginInit(ctx unsafe.Pointer) int { @@ -25,7 +25,7 @@ func FLBPluginInit(ctx unsafe.Pointer) int { return output.FLB_OK } -//export FLBPluginFlush +// FLBPluginFlush flushes the data in the stream func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { var count int var ret int From ebbf2e1cce475faca898ab38ca0929c33c6ae616 Mon Sep 17 00:00:00 2001 From: r-dilip Date: Wed, 12 Sep 2018 17:35:21 -0700 Subject: [PATCH 7/8] PR Feedback --- installer/conf/out_oms.conf | 4 +- source/code/go/src/plugins/oms.go | 112 ++++++++++++++------------ source/code/go/src/plugins/out_oms.go | 5 +- 3 files changed, 67 insertions(+), 54 deletions(-) diff --git a/installer/conf/out_oms.conf b/installer/conf/out_oms.conf index c69436a1e..d4b797757 100644 --- a/installer/conf/out_oms.conf +++ b/installer/conf/out_oms.conf @@ -1,4 +1,6 @@ omsadmin_conf_path=/etc/opt/microsoft/omsagent/conf/omsadmin.conf cert_file_path=/etc/opt/microsoft/omsagent/certs/oms.crt key_file_path=/etc/opt/microsoft/omsagent/certs/oms.key -container_host_file_path=/var/opt/microsoft/docker-cimprov/state/containerhostname \ No newline at end of file +container_host_file_path=/var/opt/microsoft/docker-cimprov/state/containerhostname +container_inventory_refresh_interval=60 +kube_system_containers_refresh_interval=300 diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 8900e4ac4..bdc704798 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "os" + "strconv" "strings" "sync" "time" @@ -25,6 +26,9 @@ const DataType = "CONTAINER_LOG_BLOB" // IPName for Container Log const IPName = "Containers" +const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" +const defaultContainerInventoryRefreshInterval = 60 +const defaultKubeSystemContainersRefreshInterval = 300 var ( // PluginConfiguration the plugins configuration @@ -44,6 +48,9 @@ var ( NameIDMap map[string]string // IgnoreIDSet set of container Ids of kube-system pods IgnoreIDSet map[string]bool + + // DataUpdateMutex read and write mutex access to the container id set + DataUpdateMutex = &sync.Mutex{} ) var ( @@ -54,14 +61,6 @@ var ( Log = FLBLogger.Printf ) -// Log wrapper to Log with timestamp -// func Log(format string, v ...interface{}) { -// FLBLogger.SetPrefix(time.Now().Format("2006-01-02 15:04:05") + " : ") -// FLBLogger.Printf(format, v) -// } - -const containerInventoryPath = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory" - // ContainerInventory represents the container info type ContainerInventory struct { ElementName string `json:"ElementName"` @@ -106,17 +105,15 @@ type ContainerLogBlob struct { func populateMaps() { Log("Updating ImageIDMap and NameIDMap") - var mutex = &sync.Mutex{} + + _imageIDMap := make(map[string]string) + _nameIDMap := make(map[string]string) files, err := ioutil.ReadDir(containerInventoryPath) if err != nil { Log("error when reading container inventory %s\n", err.Error()) } - mutex.Lock() - ImageIDMap = make(map[string]string) - NameIDMap = make(map[string]string) - for _, file := range files { fullPath := fmt.Sprintf("%s/%s", containerInventoryPath, file.Name()) fileContent, err := ioutil.ReadFile(fullPath) @@ -131,16 +128,21 @@ func populateMaps() { Log("Unmarshall error when reading file %s %s \n", fullPath, unmarshallErr.Error()) } - ImageIDMap[file.Name()] = containerInventory.Image - NameIDMap[file.Name()] = containerInventory.ElementName + _imageIDMap[file.Name()] = containerInventory.Image + _nameIDMap[file.Name()] = containerInventory.ElementName } - mutex.Unlock() + Log("Locking to update image and name maps") + DataUpdateMutex.Lock() + ImageIDMap = _imageIDMap + NameIDMap = _nameIDMap + DataUpdateMutex.Unlock() + Log("Unlocking after updating image and name maps") } func createLogger() *log.Logger { var logfile *os.File - path := "/var/log/fluent-bit-out-oms-runtime.log" + path := "/var/opt/microsoft/docker-cimprov/log/fluent-bit-out-oms-runtime.log" if _, err := os.Stat(path); err == nil { fmt.Printf("File Exists. Opening file in append mode...\n") logfile, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) @@ -171,33 +173,49 @@ func createLogger() *log.Logger { return logger } -func initMaps() { +func updateContainersData() { + + containerInventoryRefreshInterval, err := strconv.Atoi(PluginConfiguration["container_inventory_refresh_interval"]) + if err != nil { + Log("Error Reading Container Inventory Refresh Interval %s", err.Error()) + containerInventoryRefreshInterval = defaultContainerInventoryRefreshInterval + } + Log("containerInventoryRefreshInterval = %d \n", containerInventoryRefreshInterval) + go initMaps(containerInventoryRefreshInterval) + + kubeSystemContainersRefreshInterval, err := strconv.Atoi(PluginConfiguration["kube_system_containers_refresh_interval"]) + if err != nil { + Log("Error Reading Kube System Container Ids Refresh Interval %s", err.Error()) + kubeSystemContainersRefreshInterval = defaultKubeSystemContainersRefreshInterval + } + Log("kubeSystemContainersRefreshInterval = %d \n", kubeSystemContainersRefreshInterval) + + go updateIgnoreContainerIds(kubeSystemContainersRefreshInterval) +} + +func initMaps(refreshInterval int) { ImageIDMap = make(map[string]string) NameIDMap = make(map[string]string) populateMaps() - for range time.Tick(time.Second * 60) { + for range time.Tick(time.Second * time.Duration(refreshInterval)) { populateMaps() } } -func updateIgnoreContainerIds() { +func updateIgnoreContainerIds(refreshInterval int) { IgnoreIDSet = make(map[string]bool) updateKubeSystemContainerIDs() - for range time.Tick(time.Second * 300) { + for range time.Tick(time.Second * time.Duration(refreshInterval)) { updateKubeSystemContainerIDs() } } func updateKubeSystemContainerIDs() { - var mutex = &sync.Mutex{} - mutex.Lock() - IgnoreIDSet = make(map[string]bool) - if strings.Compare(os.Getenv("DISABLE_KUBE_SYSTEM_LOG_COLLECTION"), "true") != 0 { Log("Kube System Log Collection is ENABLED.") return @@ -219,12 +237,19 @@ func updateKubeSystemContainerIDs() { Log("Error getting pods %s\n", err.Error()) } + _ignoreIDSet := make(map[string]bool) for _, pod := range pods.Items { for _, status := range pod.Status.ContainerStatuses { - IgnoreIDSet[status.ContainerID[9:len(status.ContainerID)]] = true + lastSlashIndex := strings.LastIndex(status.ContainerID, "/") + _ignoreIDSet[status.ContainerID[lastSlashIndex+1:len(status.ContainerID)]] = true } } - mutex.Unlock() + + Log("Locking to update kube-system container IDs") + DataUpdateMutex.Lock() + IgnoreIDSet = _ignoreIDSet + DataUpdateMutex.Unlock() + Log("Unlocking after updating kube-system container IDs") } // PostDataHelper sends data to the OMS endpoint @@ -232,12 +257,12 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { start := time.Now() var dataItems []DataItem + DataUpdateMutex.Lock() + for _, record := range tailPluginRecords { - //id := toString(record["Id"]) containerID := getContainerIDFromFilePath(toString(record["Filepath"])) - // if Id is in the list of Ids to drop (Kube-system containers) continue if containsKey(IgnoreIDSet, containerID) { continue } @@ -258,6 +283,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { mapstructure.Decode(stringMap, &dataItem) dataItems = append(dataItems, dataItem) } + DataUpdateMutex.Unlock() if len(dataItems) > 0 { logEntry := ContainerLogBlob{ @@ -270,15 +296,21 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { req.Header.Set("Content-Type", "application/json") resp, err := HTTPClient.Do(req) + elapsed := time.Since(start) + if err != nil { Log("Error when sending request %s \n", err.Error()) + Log("Failed to flush %d records after %s", len(dataItems), elapsed) + return output.FLB_RETRY } if resp == nil || resp.StatusCode != 200 { + if resp != nil { + Log("Status %s Status Code %d", resp.Status, resp.StatusCode) + } return output.FLB_RETRY } - elapsed := time.Since(start) Log("Successfully flushed %d records in %s", len(dataItems), elapsed) } @@ -290,26 +322,6 @@ func containsKey(currentMap map[string]bool, key string) bool { return c } -func readConfig() { - workspaceIDFile := "/shared/data/workspaceId" - workspaceID, err := ioutil.ReadFile(workspaceIDFile) - if err != nil { - Log("Error when reading workspaceId file") - } - - OMSEndpoint = fmt.Sprintf("https://%s.ods.opinsights.azure.com/OperationalData.svc/PostJsonDataItems", strings.TrimSpace(string(workspaceID))) - Log("OMSEndpoint %s \n\n", OMSEndpoint) - - containerHostNameFile := "/var/opt/microsoft/docker-cimprov/state/containerhostname" - containerHostName, err := ioutil.ReadFile(containerHostNameFile) - if err != nil { - Log("Error when reading containerHostName file") - } - - Computer = toString(containerHostName) - Log("Computer == %s \n\n", Computer) -} - func toString(s interface{}) string { value := s.([]uint8) return string([]byte(value[:])) diff --git a/source/code/go/src/plugins/out_oms.go b/source/code/go/src/plugins/out_oms.go index f15022ae2..dad0ede81 100644 --- a/source/code/go/src/plugins/out_oms.go +++ b/source/code/go/src/plugins/out_oms.go @@ -20,8 +20,7 @@ func FLBPluginInit(ctx unsafe.Pointer) int { Log("Initializing out_oms go plugin for fluentbit") PluginConfiguration = ReadConfig("/etc/opt/microsoft/docker-cimprov/out_oms.conf") CreateHTTPClient() - go initMaps() - go updateIgnoreContainerIds() + updateContainersData() return output.FLB_OK } @@ -49,7 +48,7 @@ func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { return PostDataHelper(records) } -//export FLBPluginExit +// FLBPluginExit exits the plugin func FLBPluginExit() int { return output.FLB_OK } From 12c8bab77ca1641ff5af2c8b365f5b09af7dbb4a Mon Sep 17 00:00:00 2001 From: r-dilip Date: Fri, 14 Sep 2018 02:43:31 -0700 Subject: [PATCH 8/8] Fixing the newline bug for Computer, and changing containerId to Id --- source/code/go/src/plugins/oms.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index bdc704798..49472c74b 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -87,7 +87,7 @@ type DataItem struct { LogEntry string `json:"LogEntry"` LogEntrySource string `json:"LogEntrySource"` LogEntryTimeStamp string `json:"LogEntryTimeStamp"` - ContainerID string `json:"ContainerId"` + ID string `json:"Id"` Image string `json:"Image"` Name string `json:"Name"` SourceSystem string `json:"SourceSystem"` @@ -277,6 +277,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap[strKey] = strValue } + stringMap["Id"] = containerID stringMap["Image"] = ImageIDMap[containerID] stringMap["Name"] = NameIDMap[containerID] stringMap["Computer"] = Computer @@ -348,7 +349,7 @@ func ReadConfig(pluginConfPath string) map[string]string { Log("Error when reading containerHostName file %s", err.Error()) } - Computer = toString(containerHostName) + Computer = strings.TrimSuffix(toString(containerHostName), "\n") Log("Computer == %s \n", Computer) OMSEndpoint = omsadminConf["OMS_ENDPOINT"]