diff --git a/Dashboard/pom.xml b/Dashboard/pom.xml index 6e391f160e74da2c7c9dcdae68eafaaf58c0517e..b921f02789475f68a1da26093e99b0abc6e9b6a6 100644 --- a/Dashboard/pom.xml +++ b/Dashboard/pom.xml @@ -55,11 +55,6 @@ zstd-jni 1.5.0-4 - - com.jd.ump - profiler - 6.2.13 - com.alibaba fastjson diff --git a/Dashboard/src/main/resources/logback-spring.xml b/Dashboard/src/main/resources/logback-spring.xml index dce10afa58762b12e140c44c2cfc275dda0cf4ef..7202d5f34157b439b41614241a950ceacae66ab0 100644 --- a/Dashboard/src/main/resources/logback-spring.xml +++ b/Dashboard/src/main/resources/logback-spring.xml @@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息， --> - + diff --git a/client/pom.xml b/client/pom.xml index 6e43109fada9e5eb79201eb8799f212b4ed992cd..47d1a0ed8d2db0088f7b18d8a7002f6ccfcb59a7 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,6 +18,32 @@ 1.4-SNAPSHOT + + + + + + com.jd.platfrom.jlog + config-apollo + 1.4-SNAPSHOT + + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index f12bafd0b88b6b72c55f4f020c9bf775b580b9b1..32f41c3cccc548f1b07961bf02993d21f5a1aaa0 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -1,11 +1,16 @@ package com.jd.platform.jlog.client; -import com.jd.platform.jlog.client.etcd.EtcdConfigFactory; -import com.jd.platform.jlog.client.etcd.EtcdStarter; + +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.mdc.Mdc; -import com.jd.platform.jlog.client.udp.HttpSender; -import com.jd.platform.jlog.client.udp.UdpClient; -import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.client.task.Monitor; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.core.ClientHandlerBuilder; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * TracerClientStarter @@ -14,15 +19,22 @@ import com.jd.platform.jlog.client.udp.UdpSender; * @date 2021-08-13 */ public class TracerClientStarter { - /** - * etcd地址 - */ - private String etcdServer; + + private final static Logger LOGGER = LoggerFactory.getLogger(TracerClientStarter.class); + + /** * 机房 */ private Mdc mdc; + /** + * 如果直接配置在app.properties/yaml等主配置文件 + * 可以用ConfigurationProperties直接获取有值对象 + */ + private TagConfig tagConfig; + + /** * TracerClientStarter */ @@ -35,8 +47,8 @@ public class TracerClientStarter { public static class Builder { private String appName; - private String etcdServer; private Mdc mdc; + private TagConfig tagConfig; public Builder() { } @@ -51,40 +63,48 @@ public class TracerClientStarter { return this; } - public Builder setEtcdServer(String etcdServer) { - this.etcdServer = etcdServer; + public Builder setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; return this; } public TracerClientStarter build() { TracerClientStarter tracerClientStarter = new TracerClientStarter(appName); - tracerClientStarter.etcdServer = etcdServer; + tracerClientStarter.tagConfig = tagConfig; tracerClientStarter.mdc = mdc; - return tracerClientStarter; } } /** - * 启动监听etcd + * 启动监听 */ public void startPipeline() { - //设置ConfigCenter - EtcdConfigFactory.buildConfigCenter(etcdServer); + // 初始化配置 + initJLogConfig(); Context.MDC = mdc; - EtcdStarter starter = new EtcdStarter(); - //与etcd相关的监听都开启 + Monitor starter = new Monitor(); starter.start(); - UdpClient udpClient = new UdpClient(); + /* UdpClient udpClient = new UdpClient(); udpClient.start(); //开启发送 UdpSender.uploadToWorker(); //开启大对象http发送 - HttpSender.uploadToWorker(); + HttpSender.uploadToWorker();*/ + } + + + /** + * 如果未赋值，从配置器获取，底层是Properties + */ + private void initJLogConfig(){ + LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig)); + Configurator configurator = ConfiguratorFactory.getInstance(); + ClientHandlerBuilder.buildHandler(tagConfig, configurator); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java deleted file mode 100644 index b2386b7fe8f822a9a59e59b54b2738302f52ecf0..0000000000000000000000000000000000000000 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.jd.platform.jlog.client.etcd; - - -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; - -/** - * @author wuweifeng wrote on 2020-01-07 - * @version 1.0 - */ -public class EtcdConfigFactory { - private static IConfigCenter configCenter; - - private EtcdConfigFactory() {} - - public static IConfigCenter configCenter() { - return configCenter; - } - - public static void buildConfigCenter(String etcdServer) { - //连接多个时，逗号分隔 - configCenter = JdEtcdBuilder.build(etcdServer); - } -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java similarity index 90% rename from client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java rename to client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 595dae500a5ef50ecf4838e8d07849c038988cc5..2fd16ba161b9b751c3b8b8aaba657598bd1e3a99 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -8,7 +8,7 @@ import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; -import com.jd.platform.jlog.common.utils.ZstdUtils; +import com.jd.platform.jlog.core.ClientHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,14 +20,14 @@ import java.io.PrintWriter; import java.util.*; /** - * UserFilter + * HttpFilter * http://blog.chinaunix.net/uid-20783755-id-4729930.html * * @author wuweifeng * @version 1.0 * @date 2021-08-16 */ -public class UserFilter implements Filter { +public class HttpFilter implements Filter { /** * 获取切量百分比的 */ @@ -38,11 +38,11 @@ public class UserFilter implements Filter { /** * 传入百分比实现类 */ - public UserFilter(ITracerPercent iTracerPercent) { + public HttpFilter(ITracerPercent iTracerPercent) { this.iTracerPercent = iTracerPercent; } - public UserFilter() { + public HttpFilter() { iTracerPercent = new DefaultTracerPercentImpl(); } @@ -127,12 +127,14 @@ public class UserFilter implements Filter { filterChain.doFilter(servletRequest, mResp); byte[] contentBytes = mResp.getContent(); String content = new String(contentBytes); - //最终的要发往worker的response，经历了base64压缩 - byte[] bytes = ZstdUtils.compress(contentBytes); - byte[] base64Bytes = Base64.getEncoder().encode(bytes); - Map responseMap = new HashMap<>(8); - responseMap.put("response", base64Bytes); + Map map = new HashMap<>(1); + map.put("errno", 200); + ClientHandler.Outcome outcome = ClientHandler.processResp(contentBytes, map); + responseMap.put("response", outcome.getContent()); + if(!outcome.getMap().isEmpty()){ + responseMap.putAll(outcome.getMap()); + } tracerObject.add(responseMap); //此处可以对content做处理,然后再把content写回到输出流中 @@ -150,7 +152,6 @@ public class UserFilter implements Filter { long tracerId, String uri) { //request的各个入参 Map params = servletRequest.getParameterMap(); - //将request信息保存 Map requestMap = new HashMap<>(params.size()); for (String key : params.keySet()) { requestMap.put(key, params.get(key)[0]); @@ -159,7 +160,7 @@ public class UserFilter implements Filter { requestMap.put("serverIp", IpUtils.getIp()); requestMap.put("tracerId", tracerId); requestMap.put("uri", uri); - tracerObject.add(requestMap); + tracerObject.add(ClientHandler.processReq(requestMap)); } @Override diff --git a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java b/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java deleted file mode 100644 index 27e2559f4db55b7d47e7993acec031eb9d1a049f..0000000000000000000000000000000000000000 --- a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.jd.platform.jlog.client.jsf; - - -/** - * jsf请求入参包装对象 - * @author wuweifeng - * @version 1.0 - * @date 2021-10-26 - */ -public interface JsfRequest { - /** - * 请求的完整入参 - */ - String requestContent(); - - /** - * 接口名 - */ - String uri(); - - /** - * 用户pin - */ - default String pin() { - return ""; - } - - /** - * 用户uuid - */ - default String uuid() { - return ""; - } - - /** - * 用户ip - */ - default String userIp() { - return ""; - } -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java b/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java deleted file mode 100644 index faa0dc92d8dfd920b486699c8b73724c22a3bcca..0000000000000000000000000000000000000000 --- a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.jd.platform.jlog.client.jsf; - -import com.jd.platform.jlog.client.Context; -import com.jd.platform.jlog.client.tracerholder.TracerHolder; -import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.model.TracerBean; -import com.jd.platform.jlog.common.utils.IdWorker; -import com.jd.platform.jlog.common.utils.IpUtils; -import com.jd.platform.jlog.common.utils.ZstdUtils; - -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** - * JsfTracer工具类 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-10-26 - */ -public class JsfTracer { - - /** - * 临时保存入参 - */ - private static Map TEMP_HOLDER = new HashMap<>(128); - - /** - * 跟踪开启 - */ - public static void begin(JsfRequest jsfRequest) { - long tracerId = IdWorker.nextId(); - TracerHolder.setTracerId(tracerId); - - TracerBean tracerBean = new TracerBean(); - tracerBean.setCreateTime(System.currentTimeMillis()); - - List> tracerObject = new ArrayList<>(); - tracerBean.setTracerObject(tracerObject); - - //将request信息保存 - Map requestMap = new HashMap<>(); - requestMap.put("appName", Context.APP_NAME); - requestMap.put("serverIp", IpUtils.getIp()); - - requestMap.put("tracerId", tracerId); - requestMap.put("uri", jsfRequest.uri().trim()); - //将用户整个request都放进去 - requestMap.put("wholeRequest", jsfRequest.requestContent()); - - requestMap.put("pin", jsfRequest.pin()); - requestMap.put("uuid", jsfRequest.uuid()); - requestMap.put("userIp", jsfRequest.userIp()); - - tracerObject.add(requestMap); - - tracerBean.setTracerId(tracerId + ""); - - TEMP_HOLDER.put(tracerId, tracerBean); - } - - /** - * 记录请求出入参，发送到worker - */ - public static void end(String response) { - long tracerId = TracerHolder.getTracerId(); - TracerBean tracerBean = TEMP_HOLDER.get(tracerId); - if (tracerBean == null) { - return; - } - - if (response != null) { - //最终的要发往worker的response，经历了base64压缩 - byte[] bytes = ZstdUtils.compress(response.getBytes(StandardCharsets.UTF_8)); - byte[] base64Bytes = Base64.getEncoder().encode(bytes); - Map responseMap = new HashMap<>(8); - responseMap.put("response", base64Bytes); - - List> tracerObject = tracerBean.getTracerObject(); - tracerObject.add(responseMap); - } - - //设置耗时 - tracerBean.setCostTime((int) (System.currentTimeMillis() - tracerBean.getCreateTime())); - UdpSender.offerBean(tracerBean); - - //从缓存删除它 - TEMP_HOLDER.remove(tracerId); - } - -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java similarity index 50% rename from client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java rename to client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java index f7eb6702eb8b2917d46e2a2f8ce8e24bb886bb43..991a7381d5a7d7af4e45747d3101290c171a88a2 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java @@ -1,41 +1,34 @@ -package com.jd.platform.jlog.client.etcd; +package com.jd.platform.jlog.client.task; + -import cn.hutool.core.collection.CollectionUtil; -import com.ibm.etcd.api.KeyValue; -import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.worker.WorkerInfoHolder; -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.constant.Constant; -import io.grpc.StatusRuntimeException; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** - * etcd连接管理器 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-08-13 + * @author tangbohu + * @version 1.0.0 + * @ClassName Watchdog.java + * @Description TODO + * @createTime 2022年02月12日 10:20:00 */ -public class EtcdStarter { - /** - * logger - */ - private Logger logger = LoggerFactory.getLogger(getClass()); +public class Monitor { + + private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class); /** * 开始获取workerIp地址并保存 * 监听workerIp地址变化 */ public void start() { - fetchWorkerInfo(); + //fetchWorkerInfo(); } /** @@ -45,8 +38,12 @@ public class EtcdStarter { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 scheduledExecutorService.scheduleAtFixedRate(() -> { - logger.info("trying to connect to etcd and fetch worker info"); - fetch(); + LOGGER.info("trying to connect to etcd and fetch worker info"); + try { + fetch(); + } catch (Exception e) { + e.printStackTrace(); + } }, 0, 30, TimeUnit.SECONDS); } @@ -54,39 +51,22 @@ public class EtcdStarter { /** * 从配置中心获取worker的ip集合 */ - private void fetch() { - IConfigCenter configCenter = EtcdConfigFactory.configCenter(); + private void fetch() throws Exception { + Configurator configurator = ConfiguratorFactory.getInstance(); //获取所有worker的ip - List keyValues = null; + List addresses; try { //如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有 - if (Context.MDC != null) { - String mdc = parseMdc(Context.MDC); - keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); - } - if (CollectionUtil.isEmpty(keyValues)) { - keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME); - } - + addresses = configurator.getList("workers"); //全是空，给个警告 - if (CollectionUtil.isEmpty(keyValues)) { - logger.warn("very important warn !!! workers ip info is null!!!"); + if (addresses == null || addresses.size() == 0) { + LOGGER.warn("very important warn !!! workers ip info is null!!!"); + return; } - - List addresses = new ArrayList<>(); - if (keyValues != null) { - for (KeyValue keyValue : keyValues) { - //value里放的是ip地址 - String ipPort = keyValue.getValue().toStringUtf8(); - addresses.add(ipPort); - } - } - //将对应的worker保存下来 WorkerInfoHolder.mergeAndConnectNew(addresses); - } catch (StatusRuntimeException ex) { - //etcd连不上 - logger.error("etcd connected fail. Check the etcd address!!!"); + } catch (Exception ex) { + LOGGER.error("config server connected fail. Check the config address!!!"); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java index e947662e933f6cff2d162436fce92c031bf1de43..af4645b4f1bd1f6cc980088072985016cb9c3d28 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java @@ -60,7 +60,7 @@ public class UdpClient { }); //4.bind并返回一个channel try { - Channel channel = bootstrap.bind(8888).sync().channel(); + Channel channel = bootstrap.bind(8887).sync().channel(); Context.CHANNEL = channel; //6.等待channel的close @@ -68,6 +68,8 @@ public class UdpClient { //7.关闭group group.shutdownGracefully(); } catch (InterruptedException e) { + System.out.println("NioEventLoopGroup ==> "+e.toString()); + e.printStackTrace(); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java index c032766ea7cad0d6a68a8e3fe6a2ceb3ae51310b..c5631816bcef7b4c50b69a1eab275ccca771812c 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java @@ -1,12 +1,11 @@ package com.jd.platform.jlog.client.udp; -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.collect.Queues; import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.model.TracerData; import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.AsyncWorker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,8 +112,8 @@ public class UdpSender { try { //要么key达到500个，要么达到1秒，就汇总上报给worker一次 List tempLogs = new ArrayList<>(); - Queues.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempLogs)) { + AsyncWorker.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS); + if (tempLogs.size() == 0) { continue; } diff --git a/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java b/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java index 73dff8a3ab9d32a67e39a5029b873e338c1509e6..e10f6992c2dded7bf37272cc9f2544c0b96a5495 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java +++ b/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.client.worker; -import cn.hutool.core.collection.CollectionUtil; import java.util.Collections; import java.util.List; @@ -57,7 +56,7 @@ public class WorkerInfoHolder { * address例子：10.12.139.152:11111 */ public static void mergeAndConnectNew(List allAddresses) { - if (CollectionUtil.isEmpty(allAddresses)) { + if (allAddresses.size() == 0) { WORKER_HOLDER.clear(); return; } diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml deleted file mode 100644 index bf94e96df7503a2b099b4045c30132fed8571b7d..0000000000000000000000000000000000000000 --- a/clientdemo/src/main/resources/application.yml +++ /dev/null @@ -1,5 +0,0 @@ -#etcd的地址，如有多个用逗号分隔 -config: - server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ -server: - port: 8081 \ No newline at end of file diff --git a/clientdemo/src/main/resources/log4j2.xml b/clientdemo/src/main/resources/log4j2.xml deleted file mode 100644 index 00f41b939d76b720963b65e845ed8729944bb241..0000000000000000000000000000000000000000 --- a/clientdemo/src/main/resources/log4j2.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml index 607e28555841b9a4432a14d3f0372b4b85c5bff7..1999299cfbf1d4427e180acf963eea095b766f97 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -17,10 +17,12 @@ 1.1.8.4 1.5.0-4 1.2.70 + 1.7.2 + + @@ -29,6 +31,7 @@ slf4j-log4j12 1.7.32 + io.protostuff protostuff-core @@ -40,41 +43,30 @@ ${protostuff.version} - - cn.hutool - hutool-all - ${hutool.version} - - - com.github.luben - zstd-jni - ${zstd.version} - org.xerial.snappy snappy-java ${snappy.version} + - io.netty - netty-all - ${netty4.version} + com.github.luben + zstd-jni + ${zstd.version} + com.alibaba fastjson ${fastjson.version} + - com.ibm.etcd - etcd-java - ${hp-etcd.version} - - - com.github.ben-manes.caffeine - guava - ${caffeine.version} + io.netty + netty-all + ${netty4.version} + \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java deleted file mode 100644 index d5e28b617e43872fc09016342a458134e137e88c..0000000000000000000000000000000000000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.jd.platform.jlog.common.config; - -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.client.kv.KvClient; - -import java.util.List; - -/** - * 配合中心接口 - * - * @author wuweifeng wrote on 2019-12-09 - * @version 1.0 - */ -public interface IConfigCenter { - /** - * 存入key，value - */ - void put(String key, String value); - - /** - * 存入key、value，和租约id - */ - void put(String key, String value, long leaseId); - - /** - * 删除一个lease - */ - void revoke(long leaseId); - - /** - * 存入key、value，和过期时间，单位是秒 - */ - long putAndGrant(String key, String value, long ttl); - - /** - * 给key设置新的leaseId - */ - long setLease(String key, long leaseId); - - void delete(String key); - - /** - * 根据key，获取value - */ - String get(String key); - - /** - * 获取指定前缀的所有key-value - */ - List getPrefix(String key); - - /** - * 监听key - */ - KvClient.WatchIterator watch(String key); - - /** - * 监听前缀为key的 - */ - KvClient.WatchIterator watchPrefix(String key); - - /** - * 自动续约 - * @param frequencySecs 续约频率，最小是4秒，默认是5秒 - * @param minTtl 最小存活时间，最小是2秒，默认是10秒 - * @return 返回leaseId - */ - long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception; - - /** - * 构建一个可自动续约的lease - */ - long buildAliveLease(int frequencySecs, int minTtl) throws Exception; - - /** - * 构建一个普通lease - */ - long buildNormalLease(long ttl); - - /** - * 判断剩余的过期时间 - */ - long timeToLive(long leaseId); - - /** - * 根据key，获取value - */ - KeyValue getKv(String key); -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java deleted file mode 100644 index b9fef7e1e0d04235a905092953c69313f4ec448d..0000000000000000000000000000000000000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.jlog.common.config.etcd; - -import com.ibm.etcd.client.EtcdClient; - -/** - * etcd连接器 - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class JdEtcdBuilder { - - /** - * @param endPoints 如https://127.0.0.1:2379 有多个时逗号分隔 - */ - public static JdEtcdClient build(String endPoints) { - return new JdEtcdClient(EtcdClient.forEndpoints(endPoints).withPlainText().build()); - } -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java deleted file mode 100644 index 9f2254c77d7270ec20f381d8dc6d7e0406835056..0000000000000000000000000000000000000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.jd.platform.jlog.common.config.etcd; - -import cn.hutool.core.collection.CollectionUtil; -import com.google.protobuf.ByteString; -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.LeaseGrantResponse; -import com.ibm.etcd.api.RangeResponse; -import com.ibm.etcd.client.KvStoreClient; -import com.ibm.etcd.client.kv.KvClient; -import com.ibm.etcd.client.lease.LeaseClient; -import com.ibm.etcd.client.lease.PersistentLease; -import com.ibm.etcd.client.lock.LockClient; -import com.jd.platform.jlog.common.config.IConfigCenter; - -import java.util.List; -import java.util.concurrent.ExecutionException; - -import static java.util.concurrent.TimeUnit.SECONDS; - -/** - * etcd客户端 - * - * @author wuweifeng wrote on 2019-12-06 - * @version 1.0 - */ -public class JdEtcdClient implements IConfigCenter { - - private KvClient kvClient; - private LeaseClient leaseClient; - private LockClient lockClient; - - - public JdEtcdClient(KvStoreClient kvStoreClient) { - this.kvClient = kvStoreClient.getKvClient(); - this.leaseClient = kvStoreClient.getLeaseClient(); - this.lockClient = kvStoreClient.getLockClient(); - } - - - public LeaseClient getLeaseClient() { - return leaseClient; - } - - public void setLeaseClient(LeaseClient leaseClient) { - this.leaseClient = leaseClient; - } - - public KvClient getKvClient() { - return kvClient; - } - - public void setKvClient(KvClient kvClient) { - this.kvClient = kvClient; - } - - public LockClient getLockClient() { - return lockClient; - } - - public void setLockClient(LockClient lockClient) { - this.lockClient = lockClient; - } - - @Override - public void put(String key, String value) { - kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value)).sync(); - } - - @Override - public void put(String key, String value, long leaseId) { - kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value), leaseId).sync(); - } - - @Override - public void revoke(long leaseId) { - leaseClient.revoke(leaseId); - } - - @Override - public long putAndGrant(String key, String value, long ttl) { - LeaseGrantResponse lease = leaseClient.grant(ttl).sync(); - put(key, value, lease.getID()); - return lease.getID(); - } - - @Override - public long setLease(String key, long leaseId) { - kvClient.setLease(ByteString.copyFromUtf8(key), leaseId); - return leaseId; - } - - @Override - public void delete(String key) { - kvClient.delete(ByteString.copyFromUtf8(key)).sync(); - } - - @Override - public String get(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync(); - List keyValues = rangeResponse.getKvsList(); - - if (CollectionUtil.isEmpty(keyValues)) { - return null; - } - return keyValues.get(0).getValue().toStringUtf8(); - } - - @Override - public KeyValue getKv(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync(); - List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { - return null; - } - return keyValues.get(0); - } - - @Override - public List getPrefix(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); - return rangeResponse.getKvsList(); - } - - @Override - public KvClient.WatchIterator watch(String key) { - return kvClient.watch(ByteString.copyFromUtf8(key)).start(); - } - - @Override - public KvClient.WatchIterator watchPrefix(String key) { - return kvClient.watch(ByteString.copyFromUtf8(key)).asPrefix().start(); - } - - @Override - public long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception { - //minTtl秒租期，每frequencySecs秒续约一下 - PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start(); - long newId = lease.get(3L, SECONDS); - put(key, value, newId); - return newId; - } - - @Override - public long buildAliveLease(int frequencySecs, int minTtl) throws Exception { - PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start(); - - return lease.get(3L, SECONDS); - } - - @Override - public long buildNormalLease(long ttl) { - LeaseGrantResponse lease = leaseClient.grant(ttl).sync(); - return lease.getID(); - } - - @Override - public long timeToLive(long leaseId) { - try { - return leaseClient.ttl(leaseId).get().getTTL(); - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); - return 0L; - } - } -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index 6383e213081462f716c63bc316ea1fd1a1a4a659..71dbafcce03619ad63af878ace4daa05300281c7 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -1,5 +1,8 @@ package com.jd.platform.jlog.common.constant; +import java.util.HashSet; +import java.util.Set; + /** * 常量工具类Constant * @@ -24,4 +27,54 @@ public class Constant { * 当客户端要删除某个key时，就往etcd里赋值这个value，设置1秒过期，就算删除了 */ public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; + + + /** + * 普通tag最大长度 超过则不会单独存储，不超过则会存入tag：{"normal" : "logContent"} + */ + public static int TAG_NORMAL_KEY_MAX_LEN = 20; + + /** + * 可供提取的日志最小长度 + */ + public static int EXTRACT_MIN_LEN = 5; + + /** + * 符合正则但不构成kv结构的普通日志，如「xx查询为空」 + */ + public static final String TAG_NORMAL_KEY = "normal"; + + /** + * 小于1000不考虑压缩 + */ + public static final long MIN = 1000; + + + /** + * 默认大于10000才考虑压缩 + */ + public static final long THRESHOLD = 10000; + + + /** + * 正则里需要特殊处理的字符串 + */ + public static final Set SPECIAL_CHAR = new HashSet() { + { + add("*"); + add("."); + add("?"); + add("+"); + add("$"); + add("^"); + add("["); + add("]"); + add("("); + add(")"); + add("{"); + add("}"); + add("|"); + } + }; + } diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java new file mode 100644 index 0000000000000000000000000000000000000000..da4d95422ef26ee559f2c61bd327bda3d8283a5d --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java @@ -0,0 +1,100 @@ +package com.jd.platform.jlog.common.handler; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CollectMode.java + * @Description 定义采集模式的组合 用于以后method级别的tag注解 + * @createTime 2022年02月24日 23:10:00 + */ + +public class CollectMode { + + /** + * 挂起 不提取不压缩 + */ + public static final long SUSPEND = 0L; + + /** + * 提取req + */ + public static final long E_REQ = 1L; + + /** + * 压缩req + */ + public static final long C_REQ = 1L << 2; + + /** + * 提取log + */ + public static final long E_LOG = 1L << 3; + + /** + * 压缩log + */ + public static final long C_LOG = 1L << 4; + + /** + * 提取resp + */ + public static final long E_RESP = 1L << 5; + + /** + * 压缩resp + */ + public static final long C_RESP = 1L << 6; + + + + /** ======================================= 下面是组合 ======================================= */ + + + + /** + * 提取req+log + */ + public static final long EXTRACT_REQ_LOG = E_REQ | E_LOG; + + /** + * 提取req+resp + */ + public static final long EXTRACT_REQ_REQP = E_REQ | E_RESP; + + /** + * 提取req+resp + */ + public static final long EXTRACT_REQ_RESP = E_REQ | E_RESP; + + /** + * 提取req+log+resp + */ + public static final long EXTRACT_ALL = E_REQ | E_LOG | E_RESP; + + + /** + * 压缩req+log + */ + public static final long COMPRESS_REQ_LOG = C_REQ | C_LOG; + + /** + * 压缩req+resp + */ + public static final long COMPRESS_REQ_REQP = C_REQ | C_RESP; + + /** + * 压缩log+resp + */ + public static final long COMPRESS_LOG_RESP = C_REQ | C_RESP; + + /** + * 压缩req+log+resp + */ + public static final long COMPRESS_ALL = C_REQ | C_LOG | C_RESP; + + + public static boolean isMatched(long indicator, long position) { + return (indicator & position) == position; + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..5d13f834c1532b2e0ef0afb04168b691150f8721 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -0,0 +1,97 @@ +package com.jd.platform.jlog.common.handler; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.ZstdUtils; +import com.sun.istack.internal.NotNull; + +import java.util.Base64; +import java.util.Map; + +import static com.jd.platform.jlog.common.constant.Constant.MIN; +import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD; +import static com.jd.platform.jlog.common.handler.CollectMode.*; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CompressHandler.java + * @Description TODO + * @createTime 2022年03月10日 20:52:00 + */ +public class CompressHandler { + + /** + * 压缩策略 + */ + private long compress; + + /** + * 超过limit的才压缩 + */ + private long threshold; + + + private CompressHandler(long compress, long threshold){ + this.compress = compress; + this.threshold = threshold; + } + + + private static volatile CompressHandler instance = null; + + + public static void buildCompressHandler(Long compress, Long threshold) { + if(compress == null || compress < 1){ + compress = COMPRESS_LOG_RESP; + } + if( threshold == null || threshold < MIN){ + threshold = THRESHOLD; + } + instance = new CompressHandler(compress, threshold); + } + + public static void refresh(Long compress, Long threshold){ + instance = null; + buildCompressHandler(compress, threshold); + } + + public static Map compressReq(Map map){ + if(instance == null || !isMatched(instance.compress, E_REQ)){ + return map; + } + + for (Map.Entry entry : map.entrySet()) { + map.put(entry.getKey(), doCompress(entry.getValue().toString().getBytes())); + } + return map; + } + + public static byte[] compressLog(byte[] contentBytes){ + if(instance == null || !isMatched(instance.compress, E_LOG)){ return contentBytes; } + return doCompress(contentBytes); + } + + public static byte[] compressResp(byte[] contentBytes){ + if(instance == null || !isMatched(instance.compress, E_RESP)){ return contentBytes; } + return doCompress(contentBytes); + } + + private static byte[] doCompress(byte[] contentBytes){ + if(contentBytes.length < instance.threshold){ + return contentBytes; + } + //最终的要发往worker的response，经历了base64压缩 + byte[] bytes = ZstdUtils.compress(contentBytes); + return Base64.getEncoder().encode(bytes); + } + + + public long getCompress() { + return compress; + } + + public void setCompress(long compress) { + this.compress = compress; + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..b72207672570abec21a4371bc64541ced7f8a4fd --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java @@ -0,0 +1,225 @@ +package com.jd.platform.jlog.common.handler; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.ConfigUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.sun.istack.internal.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.jd.platform.jlog.common.constant.Constant.EXTRACT_MIN_LEN; +import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY; +import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY_MAX_LEN; +import static com.jd.platform.jlog.common.handler.CollectMode.*; +import static com.jd.platform.jlog.common.handler.CollectMode.E_LOG; +import static com.jd.platform.jlog.common.handler.CollectMode.E_REQ; +import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ExtractHandler.java + * @createTime 2022年02月12日 21:28:00 + */ +public class ExtractHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExtractHandler.class); + + private List reqTags; + + private List logTags; + + private List respTags; + + private String delimiter = "|"; + + private int delimiterLen = delimiter.length(); + + private String join = "="; + + private Pattern pattern; + + private long extract; + + private static volatile ExtractHandler instance = null; + + /** + * 构建标签处理器 + * @param tagConfig 配置类 + */ + public static void buildExtractHandler(TagConfig tagConfig) { + if(tagConfig.getExtract() == SUSPEND){ + return; + } + + ExtractHandler handler = new ExtractHandler(); + handler.extract = tagConfig.getExtract(); + handler.reqTags = tagConfig.getReqTags(); + handler.logTags = tagConfig.getLogTags(); + handler.respTags = tagConfig.getRespTags(); + + String regex = tagConfig.getRegex(); + if(StringUtil.isNotBlank(regex)){ + handler.pattern = Pattern.compile(regex); + }else{ + String escapeDelimiter = ConfigUtil.escapeExprSpecialWord(tagConfig.getDelimiter()); + String str = String.format("(%s[\\w\\W]*?%s)", escapeDelimiter, escapeDelimiter); + handler.pattern = Pattern.compile(str); + } + handler.delimiter = tagConfig.getDelimiter(); + handler.delimiterLen = tagConfig.getDelimiter().length(); + handler.join = tagConfig.getJoin(); + instance = handler; + LOGGER.info("构建标签处理器单例完成:{}",instance.toString()); + } + + + /** + * 提取请求参数里的标签 + * @param reqMap 额外附加的,如ip等 + * @return tags + */ + public static Map extractReqTag(Map reqMap) { + + if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } + + System.out.println("### REQ INSTANCE :"+instance.toString()); + + Map tagMap = new HashMap<>(instance.reqTags.size()); + for (String tag : instance.reqTags) { + Object val = reqMap.get(tag); + if(val != null){ + tagMap.put(tag, val); + } + } + System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(tagMap)); + return tagMap; + } + + + /** + * 提取普通log里标签 + * @param content 内容 + * @return tags + */ + public static Map extractLogTag(String content) { + if(instance == null || !isMatched(instance.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ + return null; + } + + Map tagMap = new HashMap<>(3); + Matcher m = instance.pattern.matcher(content); + while (m.find()) { + String str = m.group().substring(instance.delimiterLen, m.group().length() - instance.delimiterLen); + if(str.contains(instance.join)){ + String[] arr = str.split(instance.join); + if(instance.logTags.contains(arr[0])){ + tagMap.put(arr[0], arr[1]); + } + }else if(str.length() < TAG_NORMAL_KEY_MAX_LEN){ + tagMap.put(TAG_NORMAL_KEY, str); + }else{ + if (RANDOM.nextInt(50) == 1) { + LOGGER.info("Some logs lack tags and are larger than 20 in length. Therefore, they are simply stored"); + } + } + } + System.out.println("提取到了请求log日志标签："+JSON.toJSONString(tagMap)); + return tagMap; + } + + + /** + * 提取返回结果的tag + * @param resp 参数 + * @return map + */ + public static Map extractRespTag(Map resp) { + + if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } + + Map requestMap = new HashMap<>(instance.respTags.size()); + for (String tag : instance.respTags) { + Object val = resp.get(tag); + if(val != null){ + requestMap.put(tag, val); + } + } + System.out.println("提取到了请求出参日志标签："+JSON.toJSONString(requestMap)); + return requestMap; + } + + + + + /** + * 刷新标签处理器 加锁是为了防止极端情况下, 先到的config1覆盖后到的config2 + * @param tagConfig 新的配置 + */ + public synchronized static void refresh(TagConfig tagConfig) { + instance = null; + buildExtractHandler(tagConfig); + } + + + @Override + public String toString() { + return "TagHandler{" + + "reqTags=" + reqTags + + ", logTags=" + logTags + + ", delimiter='" + delimiter + '\'' + + ", delimiterLen=" + delimiterLen + + ", join='" + join + '\'' + + ", pattern=" + pattern + + ", extract=" + extract + + '}'; + } + + // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); + public static List extractTest(String content) { + + TagConfig cfg = new TagConfig(); + cfg.setDelimiter("|"); + cfg.setJoin("="); + + buildExtractHandler(cfg); + + List list = new ArrayList<>(); + // Matcher m = BRACKET_PATTERN.matcher(content); + Matcher m = instance.pattern.matcher(content); + while (m.find()) { + list.add(m.group().substring(1, m.group().length() - 1)); + // list.add(m.group().substring(2, m.group().length() - 2)); + + } + return list; + } + + + //static Pattern BRACKET_PATTERN = Pattern.compile("(\\{\\{[\\w\\W]*?\\}\\})"); + // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|\\|[\\w\\W]*?\\|\\|)"); + static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); + + static String str1 = "|errno=val3||node=val4||这是随便的log|"; + static String str2 = "||a=1||b=2||qwewe||"; + static String str3 = "||a=1||eee||b=2"; + static String str4 = "||a=1||eee||b=2||"; + + + public static void main(String[] args) { + + + //### INSTANCE:TagHandler{reqTags=[uid, url], logTags=[node, bizType], + // delimiter='|', delimiterLen=1, join='"="', pattern="", extract=41} + //### .content:|errno=val3||node=val4||这是随便的log| + System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1))); + /* System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2))); + System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3))); + System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4)));*/ + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..4a8e9ab4886dec8db8b0511b4de38073f0470b6f --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java @@ -0,0 +1,93 @@ +package com.jd.platform.jlog.common.handler; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.ConfigUtil; +import com.jd.platform.jlog.common.utils.FastJsonUtils; +import com.jd.platform.jlog.common.utils.StringUtil; + +import java.lang.reflect.Field; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke; +import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName JcProperties.java + * @Description TODO + * @createTime 2022年03月07日 12:43:00 + */ +public class JcProperties extends Properties { + + public JcProperties() {} + + public String getString(String key) { + Object val = get(key); + if(val == null){ + return null; + } + return String.valueOf(val); + } + + public Long getLong(String key) { + String val = getString(key); + if(val == null){ + return null; + } + return Long.parseLong(val); + } + + + public List getStrList(String key) { + if(StringUtil.isEmpty(key)){ + return null; + } + String val = getString(key); + if(StringUtil.isNotEmpty(val)){ + return FastJsonUtils.toList(val,String.class); + } + return null; + } + + /* public List getStrList(String key) { + if(StringUtil.isEmpty(key)){ + return null; + } + int index = 0; + String suffix; + String fastSuffix; + List list = new ArrayList<>(); + do{ + suffix = "["+index+"]"; + fastSuffix = "["+(index+1)+"]"; + list.add(getString(key+suffix)); + index ++; + }while (getString(key + fastSuffix) != null); + return list; + }*/ + + public T getBean(String key, Class clz) { + T bean = FastJsonUtils.toBean(JSON.toJSONString(get(key)), clz); + if(bean != null){ + return bean; + } + try { + T instance = clz.newInstance(); + invoke(instance, this, ""); + return instance; + }catch (Exception e){ + e.printStackTrace(); + } + + return null; + } + + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java b/common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java new file mode 100644 index 0000000000000000000000000000000000000000..979bea39fd8527cc58c869e172b14283d4fbf6f4 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java @@ -0,0 +1,25 @@ +package com.jd.platform.jlog.common.handler; + +import java.lang.annotation.*; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Tag.java + * @Description TODO 后续完善使用 + * @createTime 2022年02月24日 20:21:00 + */ +@Documented +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Tag { + + boolean extractReq(); + + boolean compressReq(); + + boolean extractResp(); + + boolean compressResp(); + +} \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..1240f34c3e66fc87ef0788e72a1174a0ce546c2f --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java @@ -0,0 +1,123 @@ +package com.jd.platform.jlog.common.handler; + +import java.io.Serializable; +import java.util.List; + +import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_ALL; +import static com.jd.platform.jlog.common.handler.CollectMode.EXTRACT_ALL; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName TagConfig.java + * @createTime 2022年02月13日 22:35:00 + */ +public class TagConfig implements Serializable { + + /** + * 入参的tag + */ + private List reqTags; + + /** + * 普通日志的tag + */ + private List logTags; + + /** + * 返参的tag + */ + private List respTags; + + /** + * 自定义正则 可为空 + */ + private String regex; + + /** + * 分隔符 默认｜ + */ + private String delimiter = "|"; + + /** + * 连接符 默认= + */ + private String join = "="; + + /** + * 提取策略 + */ + private long extract = EXTRACT_ALL; + + + + public List getReqTags() { + return reqTags; + } + + public void setReqTags(List reqTags) { + this.reqTags = reqTags; + } + + public List getLogTags() { + return logTags; + } + + public void setLogTags(List logTags) { + this.logTags = logTags; + } + + public List getRespTags() { + return respTags; + } + + public void setRespTags(List respTags) { + this.respTags = respTags; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public String getJoin() { + return join; + } + + public void setJoin(String join) { + this.join = join; + } + + public long getExtract() { + return extract; + } + + public void setExtract(long extract) { + this.extract = extract; + } + + + @Override + public String toString() { + return "TagConfig{" + + "reqTags=" + reqTags + + ", logTags=" + logTags + + ", respTags=" + respTags + + ", regex='" + regex + '\'' + + ", delimiter='" + delimiter + '\'' + + ", join='" + join + '\'' + + ", extract='" + extract + '\'' + + '}'; + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java index 665c5ad6283da8895211ca73548ed08a791923c5..6af7ab79dec9d1f9bd303c9cbca725731c2fd61b 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java @@ -1,5 +1,7 @@ package com.jd.platform.jlog.common.model; +import java.util.Map; + /** * className：RunLogMessage * description： @@ -37,6 +39,11 @@ public class RunLogMessage { */ private String threadName; + /** + * 标签map + */ + private Map tagMap; + public long getTracerId() { return tracerId; } @@ -92,4 +99,12 @@ public class RunLogMessage { public void setThreadName(String threadName) { this.threadName = threadName; } + + public Map getTagMap() { + return tagMap; + } + + public void setTagMap(Map tagMap) { + this.tagMap = tagMap; + } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java index 66b2b2fb6efb54bc8e24c5ce4dcddf6f3d5eb5c1..ac6b03d01a7948c20b68e8fce1586d38b1a8065c 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java @@ -25,7 +25,7 @@ public class TracerBean implements Serializable { private String tracerId; /** * tracer对象，里面放的是List> - * 第一个元素是request对象，key为request，value为完整request入参，从中可找到pin、uuid + * 第一个元素是request对象，key为request，value为完整request入参，从中可找到uuid * 最后一个元素是response响应，key为response，value为响应值byte[]，可转为String入库 */ private List> tracerObject; diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java index bae815c65d0ae87fa3094fecb0a05df26415322c..11313ec4fcd7a8ca03b36a8e03e61cddf52143b8 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java @@ -1,7 +1,10 @@ package com.jd.platform.jlog.common.utils; +import java.util.Collection; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * 异步线程池AsyncPool @@ -19,4 +22,6 @@ public class AsyncPool { public static void shutDown() { threadPoolExecutor.shutdown(); } + + } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..5f340fd9d6490a9bfbca9299fde4c52ecfd82aed --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java @@ -0,0 +1,37 @@ +package com.jd.platform.jlog.common.utils; + +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName AsyncWorker.java + * @Description TODO + * @createTime 2022年02月21日 17:10:00 + */ +public class AsyncWorker { + + public static int drain(BlockingQueue q, Collection buffer, int numElements, long timeout, TimeUnit unit) throws Exception { + if(buffer == null){ + throw new Exception("[Assertion failed] - the buffer argument cannot be null"); + } + long deadline = System.nanoTime() + unit.toNanos(timeout); + int added = 0; + + while(added < numElements) { + added += q.drainTo(buffer, numElements - added); + if (added < numElements) { + E e = q.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS); + if (e == null) { + break; + } + buffer.add(e); + ++added; + } + } + return added; + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..87af2f6d1bb0cd0f5403f27809f865f10aac901d --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java @@ -0,0 +1,219 @@ +package com.jd.platform.jlog.common.utils; + +import com.alibaba.fastjson.JSON; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CollectionUtil.java + * @Description TODO + * @createTime 2022年02月21日 17:14:00 + */ +public class CollectionUtil { + + public static boolean isEmpty(Collection collection) { + return collection == null || collection.isEmpty(); + } + + public static boolean isNotEmpty(Collection collection) { + return collection != null && collection.isEmpty(); + } + + + public static boolean isEmpty(Map map) { + return !isNotEmpty(map); + } + + + public static boolean isNotEmpty(Map map) { + return map != null && !map.isEmpty(); + } + + + public static boolean equals(List c1, List c2) { + if (c1 == null && c2 == null) { + return true; + } else if (c1 != null && c2 != null) { + if (c1.size() != c2.size()) { + return false; + } else { + for(int i = 0; i < c1.size(); ++i) { + if (!Objects.equals(c1.get(i), c2.get(i))) { + return false; + } + } + + return true; + } + } else { + return false; + } + } + + public static boolean equals(Map c1, Map c2) { + if (c1 == null && c2 == null) { + return true; + } else if (c1 != null && c2 != null) { + if (c1.size() != c2.size()) { + return false; + } else { + Iterator var2 = c1.entrySet().iterator(); + + Object v1; + Object v2; + do { + if (!var2.hasNext()) { + return true; + } + + Map.Entry entry = (Map.Entry)var2.next(); + K k = entry.getKey(); + v1 = entry.getValue(); + v2 = c2.get(k); + } while(Objects.equals(v1, v2)); + + return false; + } + } else { + return false; + } + } + + + + + + public static Set diffKeys(Map m1, Map m2){ + + Set diff = new HashSet<>(1); + + for (Map.Entry kvEntry : m1.entrySet()) { + V val = m2.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.add(kvEntry.getKey().toString()); + } + } + + for (Map.Entry kvEntry : m2.entrySet()) { + V val = m1.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.add(kvEntry.getKey().toString()); + } + } + return diff; + } + + + + + public static HashMap diffMap(Map m1, Map m2){ + + HashMap diff = new HashMap<>(1); + + for (Map.Entry kvEntry : m1.entrySet()) { + V val = m2.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.put(kvEntry.getKey(), kvEntry.getValue()); + } + } + + for (Map.Entry kvEntry : m2.entrySet()) { + V val = m1.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.put(kvEntry.getKey(), kvEntry.getValue()); + } + } + return diff; + } + + + public static void main(String[] args) { + int fail = 0; + int count = 10000; + Set set = new HashSet<>(); + for (int i = 0; i < count; i++) { + int h = randomMac4Qemu().hashCode(); + int val = indexFor(h); + boolean r = set.add(val); + // System.out.println(val); + if(!r){ + fail++; + } + } + + int fail2 = 0; + Set set2 = new HashSet<>(); + for (int i = 0; i < count; i++) { + int h = randomMac4Qemu().hashCode(); + Random rd = new Random(h); + + int val = rd.nextInt(16385); + boolean r = set.add(val); + System.out.println(val); + if(!r){ + fail2++; + } + } + + System.out.println("11--》 "+fail); + System.out.println("22-》 "+ fail2); + } + static int indexFor(int h) { + return h & (16384-1); + } + public static String randomMac4Qemu() { + /* if(1==1){ + return getRandomIp(); + }*/ + Random random = new Random(); + String[] mac = { + /* String.format("%02x", 0x52), + String.format("%02x", 0x54), + String.format("%02x", 0x00),*/ + String.format("%02x", random.nextInt(0xff)), + String.format("%02x", random.nextInt(0xff)), + String.format("%02x", random.nextInt(0xff)) + }; + return String.join(":", mac); + } + + + public static String getRandomIp(){ + + //ip范围 + int[][] range = {{607649792,608174079},//36.56.0.0-36.63.255.255 + {1038614528,1039007743},//61.232.0.0-61.237.255.255 + {1783627776,1784676351},//106.80.0.0-106.95.255.255 + {2035023872,2035154943},//121.76.0.0-121.77.255.255 + {2078801920,2079064063},//123.232.0.0-123.235.255.255 + {-1950089216,-1948778497},//139.196.0.0-139.215.255.255 + {-1425539072,-1425014785},//171.8.0.0-171.15.255.255 + {-1236271104,-1235419137},//182.80.0.0-182.92.255.255 + {-770113536,-768606209},//210.25.0.0-210.47.255.255 + {-569376768,-564133889}, //222.16.0.0-222.95.255.255 + }; + + Random rdint = new Random(); + int index = rdint.nextInt(10); + return num2ip(range[index][0]+new Random().nextInt(range[index][1]-range[index][0])); + } + + + public static String num2ip(int ip) { + int [] b=new int[4] ; + String x = ""; + + b[0] = (int)((ip >> 24) & 0xff); + b[1] = (int)((ip >> 16) & 0xff); + b[2] = (int)((ip >> 8) & 0xff); + b[3] = (int)(ip & 0xff); + x=Integer.toString(b[0])+"."+Integer.toString(b[1])+"."+Integer.toString(b[2])+"."+Integer.toString(b[3]); + + return x; + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..48ef52f90f9ce1860acecad1bd7347d431315359 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -0,0 +1,236 @@ +package com.jd.platform.jlog.common.utils; + +import com.jd.platform.jlog.common.constant.Constant; +import com.jd.platform.jlog.common.handler.JcProperties; + +import java.beans.IntrospectionException; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigUtil.java + * @Description TODO + * @createTime 2022年02月13日 21:40:00 + */ +public class ConfigUtil { + + + private static final String SERVER_SUFFIX = "Server"; + + public static final char MID_LINE = '-'; + + public static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); + + + public static String escapeExprSpecialWord(String str) { + + if (str != null && str.length() > 0) { + for (String s : Constant.SPECIAL_CHAR) { + if (str.contains(s)) { + str = str.replace(s, "\\" + s); + } + } + } + return str; + } + + + public static String formatConfigStr(Properties properties) { + + StringBuilder sb = new StringBuilder(); + Enumeration enumeration = properties.propertyNames(); + while (enumeration.hasMoreElements()) { + String key = (String) enumeration.nextElement(); + Object property = properties.get(key); + if (property != null) { + property = String.valueOf(property); + } + sb.append(key).append("=").append(property).append("\n"); + } + return sb.toString(); + } + + + public static byte[] formatConfigByte(Properties properties) { + return formatConfigStr(properties).getBytes(); + } + + + //递归解析map对象 + public static void loadRecursion(Map map, String key, Properties props) { + map.forEach((k, v) -> { + System.out.println("k => " + k + " v.class => " + v.getClass() + " val => " + v.toString() + " 是不是爸爸 => " + isParent(v)); + if (isParent(v)) { + if (v instanceof Boolean || v instanceof List) { + props.put(key + "." + k, v); + } else { + Map nextValue = (Map) v; + loadRecursion(nextValue, (("".equals(key) ? "" : key + ".") + k), props); + } + } else { + props.put(key + "." + k, v); + } + }); + } + + //判断是否还有子节点 + public static boolean isParent(Object o) { + if (!(o instanceof String || o instanceof Character || o instanceof Byte)) { + try { + Number n = (Number) o; + // System.out.println("isParent ==> "+n); + } catch (Exception e) { + return true; + } + } + return false; + } + +/* + + public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception { + Class clz = config.getClass(); + Field[] fields = clz.getDeclaredFields(); + + for (Field field : fields) { + Method m = clz.getMethod("get" + getMethodName(field.getName())); + String val = (String)m.invoke(config); + if (val != null) { + for (ConfigCenterEnum center : ConfigCenterEnum.values()) { + String fd = field.getName().replace(SERVER_SUFFIX, ""); + if(center.name().equals(fd.toUpperCase())){ + return center; + } + } + } + } + throw new Exception("Configuration center cannot be found"); + } +*/ + + + private static String getMethodName(String fieldName) { + byte[] items = fieldName.getBytes(); + items[0] = (byte) ((char) items[0] - 'a' + 'A'); + return new String(items); + } + + + + + public static String camelToMidline(String param) { + if (param == null || "".equals(param.trim())) { + return ""; + } + int len = param.length(); + StringBuilder sb = new StringBuilder(len); + for (int i = 0; i < len; i++) { + char c = param.charAt(i); + if (Character.isUpperCase(c)) { + sb.append("-"); + sb.append(Character.toLowerCase(c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } + + public static String lowerFirst(String fromStr){ + char[] chars = fromStr.toCharArray(); + chars[0] += 32; + return String.valueOf(chars); + } + + + /** + * 只支持简单的对象形配置 + * @param model bean + * @param properties 配置 + * @param prefix 前缀 + */ + @Deprecated + public static void invoke(Object model, JcProperties properties, String prefix) throws + IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException { + + Class clz = model.getClass(); + Field[] fields = model.getClass().getDeclaredFields(); + for (Field field : fields) { + String type = field.getGenericType().toString(); + field.setAccessible(true); + + String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName())); + + prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix; + String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName(); + + switch (type){ + case "class java.lang.String": + field.set(model, properties.getString(fillName)) ; + break; + case "byte": + field.setByte(model, Byte.valueOf(properties.getString(fillName))); + break; + case "short": + field.setShort(model, Short.valueOf(properties.getString(fillName))); + break; + case "int": + field.setInt(model, Integer.parseInt(properties.getString(fillName))) ; + break; + case "long": + field.setLong(model, properties.getLong(fillName)); + break; + case "double": + field.setDouble(model, Double.valueOf(properties.getString(fillName))); + break; + case "float": + field.setFloat(model, Float.valueOf(properties.getString(fillName))); + break; + case "boolean": + field.setBoolean(model, Boolean.parseBoolean(properties.getString(fillName))); + break; + case "class java.util.Date": + Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName)); + field.set(model,date) ; + break; + default: + String tn = field.getType().getTypeName(); + if("java.util.List".equals(tn)){ + String[] arr = fillName.split("\\["); + int index = 0; + String suffix; + String fastSuffix; + List list = new ArrayList<>(); + do{ + suffix = "["+index+"]"; + fastSuffix = "["+(index+1)+"]"; + list.add(properties.getString(arr[0]+suffix)); + index ++; + }while (properties.getString(arr[0]+fastSuffix) != null); + field.set(model, list); + }else if("java.util.Map".equals(tn)){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toMap(val)); + }else if(field.getType().isArray()){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toArray(val)); + }else{ + String[] ar = type.split(" "); + Object tinyObj = Class.forName(ar[1]).newInstance(); + invoke(tinyObj, properties, prefix); + field.set(model,tinyObj); + } + } + } + + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java index 2b2ebe6a46154337a353dd389d6a8068e22554fa..976e1efacee182121986355e41deeecc476ef727 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java @@ -72,7 +72,7 @@ public class FastJsonUtils { /** * json字符串转化为map */ - public static Map stringToCollect(String s) { + public static Map toMap(String s) { return (Map) JSONObject.parseObject(s); } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java b/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java index 9cb37dc424a0efae181fb3a956989bbba1077c18..4b603051f2c13823b9b46c29a21329cda51264b0 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java @@ -1,6 +1,6 @@ package com.jd.platform.jlog.common.utils; -import com.google.common.base.Preconditions; +import com.sun.tools.javac.util.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,14 +86,14 @@ public class IdWorker { * @param workerId 工作进程Id */ private static void setWorkerId(final Long workerId) { - Preconditions.checkArgument(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE); + Assert.check(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE); IdWorker.workerId = workerId; } //下一个ID生成算法 public static long nextId() { long time = System.currentTimeMillis(); - Preconditions.checkState(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds", lastTime, time); + Assert.check(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds"+lastTime); if (lastTime == time) { if (0L == (sequence = ++sequence & SEQUENCE_MASK)) { time = waitUntilNextTime(time); diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..09aaa5f86c91e2649caf7cc05087d86956fa9f7c --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java @@ -0,0 +1,45 @@ +package com.jd.platform.jlog.common.utils; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName StringUtil.java + * @Description TODO + * @createTime 2022年02月21日 17:13:00 + */ +public class StringUtil { + + + public StringUtil() { + } + + public static boolean isEmpty(CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static boolean isNotEmpty(CharSequence cs) { + return !isEmpty(cs); + } + + + public static boolean isBlank(CharSequence cs) { + int strLen; + if (cs != null && (strLen = cs.length()) != 0) { + for(int i = 0; i < strLen; ++i) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; + } + } + + return true; + } else { + return true; + } + } + + public static boolean isNotBlank(CharSequence cs) { + return !isBlank(cs); + } + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java index 3a125d076041cfab919adfd3c3e1ed8720158341..d4757eb28d6148a989bb9429161413bd86519af1 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java @@ -2,6 +2,11 @@ package com.jd.platform.jlog.common.utils; import com.github.luben.zstd.Zstd; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + /** * zstd压缩工具类 * @@ -39,4 +44,116 @@ public class ZstdUtils { return ob; } + + + /** + * 解压 + */ + public static byte[] decompressBytes2(byte[] bytes) { + int size = (int) Zstd.decompressedSize(bytes); + System.out.println("size-> "+size); + byte[] ob = new byte[size]; + Zstd.decompress(ob, bytes); + + return ob; + } + + public static void main(String[] args) throws InterruptedException, ClassNotFoundException, IllegalAccessException, InstantiationException { + + String s = "-75"; + char firstChar = s.charAt(0); + System.out.println(firstChar); + if (firstChar < '0') { // Possible leading "+" or "-" + if (firstChar == '-') { + System.out.println("========="); + } else if (firstChar != '+') + System.out.println("eeee"); + + } + // System.out.println(Integer.parseInt("-75")); + String str1 = "1a"; + String str2 = "0a"; + + String dbStr = Arrays.toString(compress(str1.getBytes())); + System.out.println(dbStr); + String[] arr0 = dbStr.split(","); + + byte[] bt = new byte[arr0.length]; + for (int i = 0; i < arr0.length; i++) { + if(i == 0){ + String first = arr0[0].replace("[", ""); + bt[i] = Byte.valueOf(first); + continue; + } + if(i == arr0.length-1){ + String last = arr0[arr0.length-1].replace("]", ""); + bt[i] = Byte.valueOf(last.trim()); + continue; + } + System.out.println(arr0[i]+" len => "+arr0[i].length()); + bt[i] = Byte.parseByte(arr0[i].trim()); + } + + + System.out.println(Arrays.toString(bt)); + // System.out.println(Arrays.toString(compress(str2.getBytes()))); + + AtomicInteger fail = new AtomicInteger(0); + AtomicInteger ok = new AtomicInteger(0); + + for (int j = 0; j < 1; j++) { + new Thread(() -> { + for (int i = 0; i < 100; i++) { + int id = new Random().nextInt(99000000); + /* String ml = String.valueOf(getRandomChar()) +id+getRandomChar(); + String ml2 = LocalDateTime.now().toString()+ getRandomChar(); + ml = ml2 + ml;*/ + // 压缩后的 + String ml = ""+i; + byte[] mlc1= compress(ml.getBytes()); + + //存进去new String(mlc1) + byte[] resul = new String(mlc1).getBytes(); + System.out.println(Arrays.toString(new String(mlc1).getBytes())); + byte[] arr = new byte[9]; + arr[0] = 40; + arr[1] = -17; + arr[2] = -65; + arr[3] = -67; + arr[4] = 47; + arr[5] = -17; + arr[6] = -65; + arr[7] = -67; + arr[8] = 32; + // System.out.println(new String(arr)); + + if(resul[0]==40 + && resul[1]==-17 + && resul[2]==-65 + && resul[3]==-67 + && resul[4]==47 + && resul[5]==-17 + && resul[6]==-65 + && resul[7]==-67 + && resul[8]==32 ){ + ok.incrementAndGet(); + }else{ + fail.incrementAndGet(); + } + } + }).start(); + } + + + Thread.sleep(30000); + + + System.out.println("ok == > "+ok.get()); + System.out.println("fail == > "+fail.get()); + + } + + public static char getRandomChar() { + return (char) (0x4e00 + (int) (Math.random() * (0x9fa5 - 0x4e00 + 1))); + } } \ No newline at end of file diff --git a/config/config-apollo/pom.xml b/config/config-apollo/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..dc41155f523804acd1ec8261a05d128208f854b9 --- /dev/null +++ b/config/config-apollo/pom.xml @@ -0,0 +1,36 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-apollo + config-apollo + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + com.ctrip.framework.apollo + apollo-client + 1.9.2 + + + slf4j-api + org.slf4j + + + + + + + diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java new file mode 100644 index 0000000000000000000000000000000000000000..573999b82872a9275b90aa1b6b7bedc324b2a477 --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -0,0 +1,165 @@ +package com.jd.platform.jlog.config.apollo; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.*; + +import com.alibaba.fastjson.JSON; +import com.ctrip.framework.apollo.ConfigFile; +import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; +import com.ctrip.framework.apollo.core.utils.StringUtils; +import com.ctrip.framework.apollo.enums.PropertyChangeType; +import com.ctrip.framework.apollo.model.ConfigChange; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.FastJsonUtils; +import com.jd.platform.jlog.core.*; +import com.ctrip.framework.apollo.Config; +import com.ctrip.framework.apollo.ConfigService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +import static com.ctrip.framework.apollo.core.ApolloClientSystemConsts.APP_ID; +import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke; +import static com.jd.platform.jlog.config.apollo.ApolloConstant.*; +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; + + +/** + * @author tangbohu + * @version 1.0.0 + * @Description todo env and cluster + * @ClassName ApolloConfigurator.java + * @createTime 2022年02月21日 21:21:00 + */ +public class ApolloConfigurator implements Configurator { + + + private static final Logger LOGGER = LoggerFactory.getLogger(ApolloConfigurator.class); + + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + + /** + * 里面有resourceProperties 和 configProperties + */ + private static volatile Config config; + + + private static volatile ApolloConfigurator instance; + + private ApolloConfigurator() { + loadApolloServerConfig(); + if (config == null) { + synchronized (ApolloConfigurator.class) { + if (config == null) { + // apollo的监听是按照namespace维度 + config = ConfigService.getConfig(DEFAULT_NAMESPACE); + ApolloListener apolloListener = new ApolloListener(); + config.addChangeListener(changeEvent -> { + LOGGER.info("Apollo收到事件变更, keys={}", changeEvent.changedKeys()); + for (String key : changeEvent.changedKeys()) { + ConfigChange change = changeEvent.getChange(key); + ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType())); + apolloListener.onProcessEvent(event); + } + }); + } + } + } + LOGGER.info("Apollo配置器构建完成"); + } + + + public static ApolloConfigurator getInstance() { + if (instance == null) { + synchronized (ApolloConfigurator.class) { + if (instance == null) { + instance = new ApolloConfigurator(); + } + } + } + return instance; + } + + @Override + public String getString(String key) { + return config.getProperty(key, null); + } + + @Override + public Long getLong(String key) { + return config.getLongProperty(key,null); + } + + @Override + public List getList(String key) { + return FastJsonUtils.toList(config.getProperty(key,""), String.class) ; + } + + @Override + public T getObject(String key, Class clz) { + return FastJsonUtils.toBean(config.getProperty(key,""), clz); + } + + @Override + public boolean putConfig(String key, String content) { + return false; + } + + @Override + public boolean putConfig(String dataId, String content, long timeoutMills) { + return false; + } + + + @Override + public String getType() { + return "apollo"; + } + + + private ConfigChangeType getChangeType(PropertyChangeType changeType) { + switch (changeType) { + case ADDED: + return ConfigChangeType.ADD; + case DELETED: + return ConfigChangeType.DELETE; + default: + return ConfigChangeType.MODIFY; + } + } + + + private void loadApolloServerConfig() { + + Properties properties = System.getProperties(); + if (!properties.containsKey(PROP_APP_ID)) { + System.setProperty(PROP_APP_ID, FILE_CONFIG.getString(APP_ID)); + } + if (!properties.containsKey(PROP_APOLLO_META)) { + System.setProperty(PROP_APOLLO_META, FILE_CONFIG.getString(APOLLO_META)); + } + if (!properties.containsKey(PROP_APOLLO_SECRET)) { + String secretKey = FILE_CONFIG.getString(APOLLO_SECRET); + if (!StringUtils.isBlank(secretKey)) { + System.setProperty(PROP_APOLLO_SECRET, secretKey); + } + } + if (!properties.containsKey(APOLLO_CLUSTER)) { + if (!StringUtils.isBlank(FILE_CONFIG.getString(APOLLO_CLUSTER))) { + System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getString(APOLLO_CLUSTER)); + } + } + if (!properties.containsKey(APOLLO_CONFIG_SERVICE)) { + System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getString(APOLLO_CONFIG_SERVICE)); + } + } + +} diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..6fedf351f4043c51c31e4b1a9c4cb306b1c39f1a --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java @@ -0,0 +1,18 @@ +package com.jd.platform.jlog.config.apollo; + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloConfiguratorProvider.java + * @createTime 2022年02月21日 21:26:00 + */ +public class ApolloConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return ApolloConfigurator.getInstance(); + } +} + diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java new file mode 100644 index 0000000000000000000000000000000000000000..03c0faf910706de27476c617601daf6e90cd75ac --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java @@ -0,0 +1,20 @@ +package com.jd.platform.jlog.config.apollo; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloConstant.java + * @createTime 2022年02月22日 11:06:00 + */ +public class ApolloConstant { + + static final String APOLLO_META = "apolloMeta"; + static final String APOLLO_SECRET = "apolloAccesskeySecret"; + static final String APOLLO_CLUSTER = "jLog"; + static final String APOLLO_CONFIG_SERVICE = "apollo.config-service"; + static final String PROP_APP_ID = "app.id"; + static final String PROP_APOLLO_META = "apollo.meta"; + static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.config-service"; + static final String PROP_APOLLO_SECRET = "apollo.accesskey.secret"; + static final String PROP_APOLLO_CLUSTER = "apollo.cluster"; +} diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java new file mode 100644 index 0000000000000000000000000000000000000000..a550b3da588e23c6d675c83dced1cf5b9635b6fa --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java @@ -0,0 +1,19 @@ +package com.jd.platform.jlog.config.apollo; + +import com.jd.platform.jlog.core.ClientHandlerBuilder; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloListener.java + * @createTime 2022年02月22日 19:18:00 + */ +public class ApolloListener implements ConfigChangeListener { + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + ClientHandlerBuilder.refresh(); + } +} diff --git a/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000000000000000000000000000000000000..f46c96c8ffdc6fcb6706794f5312a3c4c5646489 --- /dev/null +++ b/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.config.apollo.ApolloConfiguratorProvider \ No newline at end of file diff --git a/config/config-core/pom.xml b/config/config-core/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..ca570b17d4eba305467f1b60a6560fe702b6b48d --- /dev/null +++ b/config/config-core/pom.xml @@ -0,0 +1,26 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-core + config-core + + + + com.jd.platfrom.jlog + common + 1.4-SNAPSHOT + + + org.yaml + snakeyaml + 1.28 + + + + diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..9621b0305343ea7b42693ae0713f9317e3b0234d --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java @@ -0,0 +1,63 @@ +package com.jd.platform.jlog.core; + +import com.jd.platform.jlog.common.handler.CompressHandler; +import com.jd.platform.jlog.common.handler.ExtractHandler; +import java.util.Map; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ClientHandler.java + * @createTime 2022年03月13日 16:53:00 + */ +public class ClientHandler { + + public static Map processReq(Map reqMap){ + Map map = ExtractHandler.extractReqTag(reqMap); + return CompressHandler.compressReq(map); + } + + + public static Outcome processLog(String content){ + Map tagMap = ExtractHandler.extractLogTag(content); + return new Outcome(tagMap, null); + } + + + public static Outcome processResp(byte[] resp, Map respMap){ + Map tagMap = ExtractHandler.extractRespTag(respMap); + byte[] newContent = CompressHandler.compressResp(resp); + return new Outcome(tagMap, newContent); + } + + + + + + public static class Outcome{ + + Map map; + byte[] content; + + public Outcome(Map map, byte[] content) { + this.map = map; + this.content = content; + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + public byte[] getContent() { + return content; + } + + public void setContent(byte[] content) { + this.content = content; + } + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..7af513013e717c232800c107dbf98f986d180b29 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java @@ -0,0 +1,44 @@ +package com.jd.platform.jlog.core; + +import com.jd.platform.jlog.common.handler.CompressHandler; +import com.jd.platform.jlog.common.handler.ExtractHandler; +import com.jd.platform.jlog.common.handler.TagConfig; + +import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD; +import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_LOG_RESP; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ClientHandlerBuilder.java + * @Description TODO + * @createTime 2022年03月05日 22:07:00 + */ +public class ClientHandlerBuilder { + + + public static void buildHandler(TagConfig tagConfig, Configurator configurator){ + if(tagConfig == null){ + tagConfig = buildTagConfigByConfigurator(configurator); + } + ExtractHandler.buildExtractHandler(tagConfig); + CompressHandler.buildCompressHandler(configurator.getLong("compress"), configurator.getLong("threshold")); + } + + + public static void refresh(){ + Configurator configurator = ConfiguratorFactory.getInstance(); + if(configurator == null){ + throw new RuntimeException("configurator is null"); + } + ExtractHandler.refresh(buildTagConfigByConfigurator(configurator)); + CompressHandler.refresh(configurator.getLong("compress"), configurator.getLong("threshold") ); + } + + + + + private static TagConfig buildTagConfigByConfigurator(Configurator configurator){ + return configurator.getObject("tag-config", TagConfig.class); + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java new file mode 100644 index 0000000000000000000000000000000000000000..de81b0557ea0ac25be3d5060ad7e739d2193bb15 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -0,0 +1,107 @@ +package com.jd.platform.jlog.core; + + +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigChangeEvent.java + * @createTime 2022年02月16日 22:13:00 + */ +public class ConfigChangeEvent { + + private String key; + private String oldValue; + private String newValue; + private String namespace; + private ConfigChangeType changeType; + + + public ConfigChangeEvent(){ + + } + + public ConfigChangeEvent(String key, String newValue) { + this(key, DEFAULT_NAMESPACE, null, newValue, ConfigChangeType.MODIFY); + } + + public ConfigChangeEvent(String key, String oldValue, String newValue){ + this(key, DEFAULT_NAMESPACE, oldValue, newValue, ConfigChangeType.MODIFY); + } + + public ConfigChangeEvent(String key, String namespace, String oldValue, String newValue, + ConfigChangeType type) { + this.key = key; + this.namespace = namespace; + this.oldValue = oldValue; + this.newValue = newValue; + this.changeType = type; + } + + + public String getKey() { + return key; + } + + + public ConfigChangeEvent setKey(String key) { + this.key = key; + return this; + } + + + public String getOldValue() { + return oldValue; + } + + + public ConfigChangeEvent setOldValue(String oldValue) { + this.oldValue = oldValue; + return this; + } + + + public String getNewValue() { + return newValue; + } + + + public ConfigChangeEvent setNewValue(String newValue) { + this.newValue = newValue; + return this; + } + + + public ConfigChangeType getChangeType() { + return changeType; + } + + + public ConfigChangeEvent setChangeType(ConfigChangeType changeType) { + this.changeType = changeType; + return this; + } + + + public String getNamespace() { + return namespace; + } + + + public ConfigChangeEvent setNamespace(String namespace) { + this.namespace = namespace; + return this; + } + + @Override + public String toString() { + return "ConfigChangeEvent{" + + "key='" + key + '\'' + + ", oldValue='" + oldValue + '\'' + + ", newValue='" + newValue + '\'' + + ", namespace='" + namespace + '\'' + + ", changeType=" + changeType + + '}'; + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java new file mode 100644 index 0000000000000000000000000000000000000000..9d337a2ed7a472174dccc68fa083e2cb18753a3a --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -0,0 +1,82 @@ +/* + * Copyright 1999-2019 Seata.io Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jd.platform.jlog.core; + + +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.*; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigChangeListener.java + * @createTime 2022年02月16日 12:06:00 + */ +public interface ConfigChangeListener { + + Logger LOGGER = LoggerFactory.getLogger(ConfigChangeListener.class); + + + ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(1, 1, + Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("configListener", 1)); + + + /** + * 事件具体处理 + * @param event event + */ + void onChangeEvent(ConfigChangeEvent event); + + + /** + * 处理变更事件 + * @param event 事件 + */ + default void onProcessEvent(ConfigChangeEvent event) { + LOGGER.info("通用监听器配置处理器", event.toString()); + getExecutorService().submit(() -> { + beforeEvent(); + try { + onChangeEvent(event); + } catch (Exception e) { + e.printStackTrace(); + } + afterEvent(); + }); + } + + default void onShutDown() { + getExecutorService().shutdown(); + } + + + default ExecutorService getExecutorService() { + return EXECUTOR_SERVICE; + } + + + default void beforeEvent() { + + } + + + default void afterEvent() { + + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java new file mode 100644 index 0000000000000000000000000000000000000000..acdee80a471e75466bc0b15b4644d8a4628e6517 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java @@ -0,0 +1,14 @@ +package com.jd.platform.jlog.core; + + +/** + * @author tangbohu + */ +public enum ConfigChangeType { + + ADD, + + MODIFY, + + DELETE +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java new file mode 100644 index 0000000000000000000000000000000000000000..ad4bdf46c697ab5e14fe9f7185de28b66bb38289 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -0,0 +1,69 @@ +package com.jd.platform.jlog.core; + +import java.util.List; + +/** + * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName Configurator.java + * @createTime 2022年02月15日 17:06:00 + */ +public interface Configurator { + + /** + * 获取string配置 + * @param key key + * @return val + */ + String getString(String key); + + + /** + * 获取LONG配置 + * @param key key + * @return val + */ + Long getLong(String key); + + /** + * 获取LIST类型配置 + * @param key key + * @return val + */ + List getList(String key); + + /** + * 获取实体类型配置 + * @param key key + * @return val + */ + T getObject(String key, Class clz); + + + /** + * 设置配置 + * @param key key + * @param content val + * @return content val + */ + boolean putConfig(String key, String content); + + /** + * 设置配置 + * @param key key + * @return content val + * @param timeoutMills timeoutMills + * @param content content + */ + boolean putConfig(String key, String content, long timeoutMills); + + + /** + * 获取配置器类型 + * @return string example:apollo + */ + String getType(); + + +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..5be575dfc249ca03c69f08999f42aa3f84c62d31 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -0,0 +1,77 @@ +package com.jd.platform.jlog.core; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName ConfiguratorFactory.java + * @createTime 2022年02月15日 21:54:00 + */ +public class ConfiguratorFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConfiguratorFactory.class); + + + private static volatile Configurator instance = null; + + + public static volatile Configurator base = null; + + + public static AtomicBoolean useFileConfig = new AtomicBoolean(true); + + + + public static Configurator getInstance(){ + if (instance == null) { + synchronized (Configurator.class) { + if (instance == null) { + instance = buildConfiguration(); + LOGGER.info("构建总配置器单例完成 instance 获取类型结果:{}", instance.getType()); + LOGGER.info("构建总配置器单例完成 Base 获取serverAddr结果:{}", base.getString("serverAddr")); + } + } + } + return instance; + } + + + /** + * SPI实现装载不同的配置器 + * @return 配置器 + */ + private static Configurator buildConfiguration() { + + try { + /* + 配置文件的配置器 + */ + base = new FileConfigurator(); + } catch (IOException e) { + LOGGER.info("文件配置器构建失败", e); + throw new RuntimeException("build file buildConfiguration fail", e); + } + + /* + 配置中心的配置器，如果没有就用文件配置器 + */ + ServiceLoader builders = ServiceLoader.load(ConfiguratorProvider.class); + //noinspection LoopStatementThatDoesntLoop + for (ConfiguratorProvider provider : builders) { + LOGGER.info("配置中心的配置器获取成功, 类型为:{}", provider.build().getType()); + useFileConfig.set(false); + return provider.build(); + } + return base; + } + +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..f1ebb1329600d93766e20ecb50de538f6eb2291b --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java @@ -0,0 +1,14 @@ +package com.jd.platform.jlog.core; + + +/** + * @author tangbohu + */ +public interface ConfiguratorProvider { + + /** + * 根据实现类构建配置器 + * @return 配置器 + */ + Configurator build(); +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java new file mode 100644 index 0000000000000000000000000000000000000000..e5f78e4209620431644c4ab3e8bd3a9fd7b96f16 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java @@ -0,0 +1,52 @@ +package com.jd.platform.jlog.core; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Constant.java + * @Description TODO + * @createTime 2022年02月26日 10:19:00 + */ +public class Constant { + + /** + * 监听重读配置文件间隔 单位ms + */ + static final long LISTENER_CONFIG_INTERVAL = 10000; + + static final String CONFIG_FILE_PROPERTIES = "/application.properties"; + + static final String CONFIG_FILE_YML = "/application.yml"; + + static final String JLOG_CONFIG_FILE = "/jLog.properties"; + + + static final String ENV = "env"; + + static final String YML = "yml"; + + public static final long DEFAULT_TIMEOUT = 2 * 1000; + + static final long AWAIT_TIME = 3 * 1000; + + public static final String SERVER_ADDR_KEY = "serverAddr"; + + public static final String NAMESPACE_KEY = "namespace"; + + public static final String DEFAULT_NAMESPACE = "jLog"; + + + /** + * 配置文件集合 + */ + public static final Set CONFIG_FILES = new HashSet() { + { + add(CONFIG_FILE_PROPERTIES); + add(CONFIG_FILE_YML); + add(JLOG_CONFIG_FILE); + } + }; +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java new file mode 100644 index 0000000000000000000000000000000000000000..325fb7c80323b9fc64578f476804c517b8ce48f1 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -0,0 +1,246 @@ +package com.jd.platform.jlog.core; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; +import java.net.URL; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.jd.platform.jlog.core.ConfigChangeListener.EXECUTOR_SERVICE; +import static com.jd.platform.jlog.core.ConfiguratorFactory.useFileConfig; +import static com.jd.platform.jlog.core.Constant.*; + + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileConfigurator.java + * @createTime 2022年02月17日 23:22:00 + */ +public class FileConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class); + + private static JcProperties PROPERTIES = new JcProperties(); + + private static volatile FileListener FILELISTENER = null; + + private static final Set LISTENED_FILES = new CopyOnWriteArraySet<>(); + + private final Map FILE_MODIFY_MAP = new ConcurrentHashMap<>(); + + + FileConfigurator() throws IOException{ + + String env = System.getenv(ENV); + + for (String file : CONFIG_FILES) { + String fileName = StringUtil.isEmpty(env) ? file : file + "_" + env; + URL url = this.getClass().getResource(file); + if(url == null){ + continue; + } + try (InputStream is = url.openStream()) { + JcProperties props = new JcProperties(); + if (fileName.contains(YML)) { + props.putAll(new Yaml().loadAs(is, Map.class)); + } else { + props.load(is); + } + FILE_MODIFY_MAP.put(fileName, new FileWrapper(new File(url.getFile()).lastModified(), props)); + PROPERTIES.putAll(props); + LOGGER.info("{}配置文件配置:{}", file, props.toString()); + } + } + LOGGER.info("合并后的配置:{}",PROPERTIES.toString()); + for (String file : CONFIG_FILES) { + file = StringUtil.isEmpty(env) ? file : file + "_" + env; + LISTENED_FILES.add(file); + if(FILELISTENER == null){ + synchronized (FileConfigurator.class){ + FILELISTENER = new FileListener(); + FILELISTENER.addListener(); + } + } + } + } + + + @Override + public String getString(String key) { + return PROPERTIES.getString(key); + } + + @Override + public Long getLong(String key) { + return PROPERTIES.getLong(key); + } + + @Override + public List getList(String key) { + return PROPERTIES.getStrList(key); + } + + @Override + public T getObject(String key, Class clz) { + return PROPERTIES.getBean(key, clz); + } + + @Override + public boolean putConfig(String key, String content) { return false; } + + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + return false; + } + + + @Override + public String getType() { + return "file"; + } + + + + class FileListener implements ConfigChangeListener { + + private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("fileListener", 1)); + + + FileListener() {} + + synchronized void addListener() { + FILELISTENER.onProcessEvent(new ConfigChangeEvent()); + } + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + for(; !getExecutorService().isShutdown() && !getExecutorService().isShutdown(); checkAndConfigure()) { + try { + if(!useFileConfig.get()){ + LOGGER.info("装配了配置中心, 文件配置器关闭"); + executor.shutdown(); + if(!executor.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ + // 超时的时候向线程池中所有的线程发出中断(interrupted)。 + executor.shutdownNow(); + return; + } + } + Thread.sleep(LISTENER_CONFIG_INTERVAL); + } catch (InterruptedException ignored) { + } + } + } + + @Override + public ExecutorService getExecutorService() { + return executor; + } + + } + + + + + /** + * 检测文件最后修改时间 和重载文件 + */ + private void checkAndConfigure(){ + + AtomicBoolean change = new AtomicBoolean(false); + Map newModifyMap = checkAndReload(); + + FILE_MODIFY_MAP.forEach((k, v)->{ + FileWrapper newFile = newModifyMap.get(k); + if(newFile != null && newFile.change){ + Set diffKeys = CollectionUtil.diffKeys(newFile.props, v.props); + if(!diffKeys.isEmpty()){ + change.set(true); + for (String diffKey : diffKeys) { + LOGGER.warn("文件 {} 配置变更 key={}变更事件:{}", k, diffKey, new ConfigChangeEvent(diffKey, String.valueOf(v.props.get(diffKey)), String.valueOf(newFile.props.get(diffKey)))); + } + v.props = newFile.props; + v.lastModify= newFile.lastModify; + } + } + }); + if(change.get()){ + LOGGER.debug("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); + PROPERTIES.clear(); + FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props)); + LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); + ClientHandlerBuilder.refresh(); + } + } + + + /** + * 检测文件最后修改时间 和重载文件 + */ + private Map checkAndReload() { + + Map fileWrapperMap = new ConcurrentHashMap<>(3); + + for (String fileName : LISTENED_FILES) { + URL url = this.getClass().getResource(fileName); + + if(url == null){ + continue; + } + File file = new File(url.getFile()); + long curLastMod = file.lastModified(); + FileWrapper wrapper = FILE_MODIFY_MAP.get(fileName); + Long cacheLastMod = wrapper.lastModify; + if(curLastMod <= cacheLastMod){ + continue; + } + JcProperties props = new JcProperties(); + try (InputStream is = url.openStream()) { + if (fileName.contains(YML)) { + props.putAll(new Yaml().loadAs(is, Map.class)); + } else { + props.load(is); + } + fileWrapperMap.put(fileName, new FileWrapper(curLastMod, true, props)); + LOGGER.info("定时任务监测到文件更新：{}最新配置:{}", fileName, JSON.toJSONString(props)); + } catch (Exception e) { + LOGGER.error("定时任务load失败：{}配置文件配置重载失败", fileName, e); + } + } + return fileWrapperMap; + } + + + static class FileWrapper{ + + private long lastModify; + + private boolean change; + + private JcProperties props; + + FileWrapper(long lastModify, JcProperties props) { + this.lastModify = lastModify; + this.change = false; + this.props = props; + } + FileWrapper(long lastModify, boolean change, JcProperties props) { + this.lastModify = lastModify; + this.change = change; + this.props = props; + } + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java new file mode 100644 index 0000000000000000000000000000000000000000..44b3c99c429f4d5d418c876c101ad115f0149b21 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -0,0 +1,81 @@ +package com.jd.platform.jlog.core; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.CollectMode; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.common.utils.ZstdUtils; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileNode.java + * @createTime 2022年03月08日 16:32:00 + */ +public final class FileNode { + + private String fillPath; + + private long lastModity; + + public static void main(String[] args) throws IOException { + + String a = "汉字"; + System.out.println("a byte length："+a.getBytes().length); + + byte[] bt1 = ZstdUtils.compress(a.getBytes()); + String ckStr = new String(bt1); + String newStr1 = ZstdUtils.decompress(bt1); + System.out.println("老 字符串 bt："+ Arrays.toString(a.getBytes())); + System.out.println("新 字符串 bt："+ Arrays.toString(ckStr.getBytes())); + + + System.out.println("新字符串："+newStr1); + System.out.println("新字符串的byte长度："+newStr1.getBytes().length); + + String str = "这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是"+"{\"@level\":\"info\",\"@message\":\"response json: \",\"@module\":\"testing-platform.cfeature_plugin\",\"@timestamp\":\"2022-03-10T19:37:55.181928+08:00\",这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是" + + "\"EXTRA_VALUE_AT_END\":{\"data\":{\"accountAddr\":\"bg\",\"accountArea\":\"\",\"accountBank\":\"012\",\"accountBankBranch\":\"\",\"accountBankBranchNo\":\"\",\"accountBankName\":\"BBVA BANCOMER SA\",\"accountBranch\":\"\",\"accountCardDigit\":\"\",\"accountCardType\":null,\"accountCardno\":\"012420015684144165\",\"accountCity\":\"\",\"accountCompany\":\"\",\"accountFirstName\":\"\",\"accountGrantPhoto\":{\"keys\":[\"do1_tQxmThhy446Wf8JLHK5E\"],\"urls\":[\"http://img1.didiglobal.com/static/sailing_private_b/do1_tQxmThhy446Wf8JLHK5E?expire=1647517074\\u0026signiture=iTo-tiI21Ysdn7n7Y6y2YGs36avRi0cfLUpRl0rlE-g=\"]},\"accountHolder\":\"test_accountHolder\",\"accountHolderId\":\"\",\"accountLastName\":null,\"accountType\":0,\"addr\":\"test_addr\",\"addrAlph\":\"\",\"applyId\":\"5764607808352618589\",\"applyStatus\":0,\"applyType\":1,\"areaName\":\"Area0\",\"arrBusinessAttr\":[\"140\"],\"arrTags\":[],\"authorizationInfo\":{\"payment\":{\"authorizedPics\":null," + + "\"extraFiles\":null,\"pics\":null},\"sign\":{\"authorizedPics\":null,\"extraFiles\":null,\"pics\":null}},\"autoSwitch\":\"0\",\"avgPrice\":{\"max\":0,\"min\":0},\"avgProduceTime\":600,\"bLicenseAddr\":\"\",\"bLicenseBrand\":\"\",\"bLicenseExpireFlag\":61,\"bLicenseExpireTime\":\"1970-01-01 08:00:00\",\"bLicenseId\":\"\",\"bLicenseLpName\":\"\",\"bLicensePic\":\"\",\"bLicensePicUrl\":\"\",\"bLicenseStartTime\":\"1970-01-01 08:00:00\",\"bLicenseStatus\":\"3\",\"bLicenseValidType\":0,\"bankAgentDigit\":\"\",\"bankAgentno\":\"\",\"bdName\":\"sim_data_01(simdata01_test_v)\",\"bdPhone\":\"\",\"bizDayTime\":null,\"bizDayTimeNonAffiliate\":\"\",\"bizStatus\":1,\"bizTime\":\"{\\\"day\\\":[1,1,1,1,1,1,1],\\\"time\\\":[{\\\"begin\\\":\\\"10:00\\\",\\\"end\\\":\\\"18:00\\\"}]}\",\"bizoppDocInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"" + + "licensePics\":null}},\"bizoppInfo\":{\"bussinessHours\":\"\",\"deliveryDistanceTime\":\"\",\"deliveryPrice\":0,\"itemCount\":null,\"minPrice\":0,\"priceRange\":null,\"ratingCount\":\"\",\"ratingValue\":\"\"},\"bizoppStatus\":5,\"blicenseShow\":1,\"brand\":\"\",\"brandId\":0,\"businessType\":1,\"button\":{\"addVisitButton\":1,\"applyOpenButton\":0,\"applySignButton\":0,\"approveButton\":0,\"batchAllotButton\":1,\"businessInformationViewButton\":1,\"cancelApplyButton\":0,\"conSignPadButton\":0,\"confirmButton\":0,\"controlOperationButton\":1,\"copyMsgGatherUrlButton\":0,\"deliveryPadButton\":1,\"deliverySettingButton\":1,\"editButton\":0,\"financialOperationButton\":1,\"giveUp\":0,\"msgGatherButton\":0,\"msgGatherEditButton\":0,\"msgGatherPlusButton\":0,\"orderInfoButton\":1,\"pickIn\":0,\"sendMsgGatherButton\":0,\"switchToNonAffiliateButton\":0,\"tagsButton\":1,\"topEatsButton\":0,\"updateAbnormalTagsButton\":1,\"updateAddressButton\":1,\"updateBusinessInfoButton\":1," + + "\"updateCategoryButton\":1,\"updateNormalTag\":1,\"updateShopBaseInfo\":1,\"updateShopInfo\":1,\"uploadPictureButton\":1,\"viewOpenButton\":1,\"visitListButton\":1},\"cLicenseAddr\":\"\",\"cLicenseBrand\":\"\",\"cLicenseExpireFlag\":61,\"cLicenseExpireTime\":\"1970-01-01 08:00:00\",\"cLicenseId\":\"\",\"cLicenseLpName\":\"\",\"cLicensePic\":\"\",\"cLicensePicUrl\":\"\",\"cLicenseStatus\":\"3\",\"cLicenseValidType\":0,\"category\":[],\"categoryIds\":[],\"channel\":1,\"checkStatus\":0,\"city\":52080200,\"cityId\":52080200,\"clicenseShow\":1,\"clusterId\":5764607713452294000,\"clusterName\":\"\",\"commissionJson\":[],\"competeBizTime\":[],\"competeItemMenuUrl\":[],\"completeStatus\":1,\"contactCallingCode\":\"\",\"contactName\":\"test_partyaContact\",\"contactPhone\":\"15512300000\",\"contractorId\":\"5764607748000712588\",\"contractorStatus\":3,\"cosTrans\":0,\"country\":\"MX\",\"countryCode\":\"\",\"createTime\":\"2022-02-16 19:39:42\",\"creatorUserName\":\"simdata01_test_v\"," + + "\"currency\":{\"display\":\"MX$0.00\",\"number\":\"0.00\",\"position\":0,\"sign\":\"\",\"symbol\":\"MX$\"},\"decorateId\":0,\"deliverDistance\":0,\"deliverType\":2,\"deviceFlagbin\":3,\"deviceFlagbinNeedPad\":1,\"displayable\":0,\"displayableNonAffiliate\":0,\"docFlagbin\":0,\"docId\":\"5764607687590152047\",\"docInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"licensePics\":null}},\"editTime\":\"2022-02-16 19:40:14\",\"environmentalPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"extendMaterial\":\"\",\"extraInfo\":{\"bankFlow\":{\"files\":null,\"pics\":null},\"other\":{\"files\":null,\"pics\":null}},\"filterTags\":[{\"text\":\"正常门店\",\"type\":1}],\"filterTagsValue\":0,\"flagbin\":0,\"foodSafe\":\"\",\"foodSafePic\":\"\",\"foodSafePicUrl\":\"\",\"foodSafeShow\":1," + + "\"frontPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"],\"grade\":\"\",\"headImg\":\"\",\"isDeliverCash\":1,\"isEnableCash\":1,\"isEnablePaypay\":1,\"isEnablePos\":0,\"isFoodCity\":0,\"isIllegal\":0,\"isNeedPad\":0,\"isRate\":2,\"isScopeDelivery\":1,\"itemFiles\":null,\"itemPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_iad8ykReSCUcGh7sUJye\"],\"ka\":1,\"kpContactTime\":null,\"kpDepartment\":\"\",\"kpEmail\":\"test_data_platform@didiglobal.com\",\"kpLastName\":\"\",\"kpLastNameAlph\":\"\",\"kpName\":\"test_kpName\",\"kpNameAlph\":\"\",\"kpOutId\":\"\",\"kpPhone\":\"15512300001\",\"kpPhoneExtra\":null,\"kpPics\":{\"keys\":[\"https://img0.didiglobal.com/static/soda_public/do1_BXtt9g8f8dfiecSHC1dZ\"],\"urls\":[false]},\"kpSource\":\"\",\"lat\":\"28.7370616\",\"lng\":\"-106.1306375\",\"localTaxId\":\"\",\"localTaxPayer\":\"\",\"localTaxPayerAddr\":\"\",\"localTaxPayerIdType\":0,\"localTaxPaymentMethod\":\"\"," + + "\"localTaxPics\":\"\",\"localTaxVatPayment\":0,\"logo\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"logoImg\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"lpLicenseId\":\"\",\"lpLicensePicBack\":\"\",\"lpLicensePicBackUrl\":\"\",\"lpLicensePicFront\":\"\",\"lpLicensePicFrontUrl\":\"\",\"lpLicensePics\":\"\",\"lpLicensePicsUrl\":\"\",\"lpLicenseStatus\":\"3\",\"lpLicenseType\":\"0\",\"lpName\":\"\",\"mainCategory\":\"\",\"manualCategory\":[],\"minPrice\":0,\"monthSales\":0,\"monthSold\":0,\"msgGatherUrl\":\"\",\"name\":\"test_1645011556_kFvjwPijhdxkmmgRXKC\",\"nameAlph\":\"\",\"notifyEmail\":\"\",\"notifyPhone\":\"\",\"objBusinessAttr\":[],\"objTags\":[],\"officialUrl\":\"\",\"open\":1,\"openCheck\":[{\"button\":\"已添加\",\"text\":\"联系电话\",\"type\":1},{\"button\":\"已填写\",\"text\":\"配送信息\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店头图\",\"type\":1},{\"button\":\"未完成\",\"text\":\"可售菜品数量\"," + + "\"type\":0},{\"button\":\"已通过\",\"text\":\"营业时间段\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店主营品类\",\"type\":1},{\"button\":\"已添加\",\"text\":\"出餐时长\",\"type\":1}],\"openCheckNonAffiliate\":null,\"orderConfirmMethod\":\"0\",\"padInfo\":{\"isNeedPad\":0,\"isPadOnlineSign\":1},\"padStatus\":0,\"payVerifyReason\":\"\",\"payVerifyStatus\":0,\"permAddr\":\"\",\"permNo\":\"\",\"phone\":\"164501155695\",\"poi\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiId\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiName\":\"Arcos de Terragona 16934, Villa del Nte, 31137 Chihuahua, Chih., México\",\"postCode\":\"\",\"produceTimeMax\":\"0\",\"rank\":5,\"repeatShopId\":\"\",\"reprePhone\":\"\",\"riderAreaName\":\"\",\"riderClusterId\":52080200,\"score\":0,\"secondCategory\":\"\",\"selectCityId\":52080200,\"settledStatus\":0,\"settledTags\":0,\"shopId\":\"5764607747732277132\",\"shopLayerLevel\":\"\",\"shopLevel\":\"\",\"shopOnlineStatus\":3,\"shopOnlineStatusDesc\":\"需手工开启营业\"," + + "\"shopOnlineStatusMsg\":\"歇业中\",\"shopPhone\":\"\",\"shopPics\":{\"environmentalPhoto\":[\"/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"frontPhoto\":[\"/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"]},\"shopStatus\":3,\"shopTag\":0,\"signPic\":null,\"signType\":1,\"signedTime\":1645011613,\"source\":2,\"startupCost\":500000,\"startupCostLimit\":50000,\"startupCostNum\":0,\"startupCostProportion\":1000,\"startupExecWeek\":1,\"tags\":0,\"taxId\":\"234234123112\",\"taxIdType\":1,\"taxPayer\":\"\",\"taxPayerAddr\":\"\",\"taxPayerIdType\":0,\"taxPaymentMethod\":\"\",\"taxPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_Gip4umImni45IUHPTvQQ\"],\"taxVatPayment\":0,\"thirdCategory\":\"\",\"timezone\":\"\",\"todayBizTime\":[],\"type\":1,\"updateTime\":\"2022-03-10 15:46:20\",\"visible\":false,\"visitStatus\":0},\"errmsg\":\"ok\",\"errno\":0,\"machine\":\"\",\"reqParams\":{\"locale\":\"zh-CN\",\"shopId\":\"5764607747732277132\"},\"time\":1646912275," + + "\"traceId\":\"s060310x1646912272\"},\"timestamp\":\"2022-03-10T19:37:55.180+0800\"}"; + System.out.println("字符串长度： "+str.length()); + System.out.println("字符串byte长度："+str.getBytes().length); + byte[] bt = ZstdUtils.compress(str.getBytes()); + String newStr = ZstdUtils.decompress(bt); + System.out.println("新字符串："+newStr); + System.out.println("新字符串的byte长度："+newStr.getBytes().length); + + System.out.println("Zstd压缩后的byte长度："+bt.length); + + byte[] bs = GzipCompress.compress(str.getBytes()); + System.out.println("Gzip压缩后的byte长度："+bs.length); + System.out.println("Zstd压缩后的byte长度 + Gzip压缩后的byte长度："+GzipCompress.compress(bt).length); + + + System.out.println("Base64 后的byte长度: "+Base64.getEncoder().encode(bt).length); + + System.out.println("======="); + System.out.println(CollectMode.COMPRESS_LOG_RESP); + System.out.println(CollectMode.EXTRACT_ALL); + + System.out.println(); + JcProperties properties = new JcProperties(); + String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; + properties.load(new FileInputStream(path)); + System.out.println(JSON.toJSONString(properties)); + TagConfig bean = properties.getBean("tag-config", TagConfig.class); + System.out.println(JSON.toJSONString(bean)); + } + +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java new file mode 100644 index 0000000000000000000000000000000000000000..22d75b6664dcca4cc733211f64a30f8694241d17 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java @@ -0,0 +1,49 @@ +package com.jd.platform.jlog.core; + + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * The Data Compression Based on gzip. + * + */ +public class GzipCompress { + + public static byte[] compress(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip; + + try { + gzip = new GZIPOutputStream(out); + gzip.write(data); + gzip.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + return out.toByteArray(); + } + + public static byte[] uncompress(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(data); + + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[2048]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return out.toByteArray(); + } +} diff --git a/config/config-etcd/pom.xml b/config/config-etcd/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..4c6fa9e6b3b6ef5f92c3526a3fe7433915d11994 --- /dev/null +++ b/config/config-etcd/pom.xml @@ -0,0 +1,26 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-etcd + config-etcd + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + com.ibm.etcd + etcd-java + 0.0.16 + + + + diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..09a5e2e6642a66aa9debc3532d3dc487af12ca58 --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.etcd; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +/** + * @author tangbohu + */ +public class EtcdConfigurationProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return EtcdConfigurator.getInstance(); + } +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java new file mode 100644 index 0000000000000000000000000000000000000000..85e77b25b6cd1351eb33e33d042e13a126b538b8 --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -0,0 +1,123 @@ +package com.jd.platform.jlog.etcd; + + +import java.io.IOException; +import java.io.StringReader; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import com.alibaba.fastjson.JSON; +import com.google.protobuf.ByteString; +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.api.RangeResponse; +import com.ibm.etcd.client.EtcdClient; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte; +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; +import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; +import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; + +/** + * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName EtcdConfigurator.java + * @createTime 2022年02月21日 21:46:00 + */ +public class EtcdConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(EtcdConfigurator.class); + + private static volatile EtcdConfigurator instance; + + static volatile EtcdClient client; + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + static final String ROOT = "/jLog"; + + static final String PROPERTIES_PATH = "/jLog/jLog.properties"; + + static JcProperties PROPERTIES = new JcProperties(); + + private EtcdConfigurator() { + LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getString(SERVER_ADDR_KEY)); + client = EtcdClient.forEndpoints(FILE_CONFIG.getString(SERVER_ADDR_KEY)).withPlainText().build(); + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return; + } + String val = keyValues.get(0).getValue().toStringUtf8(); + if(StringUtil.isNotBlank(val)){ + try { + PROPERTIES.load(new StringReader(val)); + } catch (IOException e) { + e.printStackTrace(); + } + } + LOGGER.info("初始化etcd配置", JSON.toJSONString(PROPERTIES)); + new EtcdListener().onProcessEvent(new ConfigChangeEvent()); + } + + + public static EtcdConfigurator getInstance() { + if (instance == null) { + synchronized (EtcdConfigurator.class) { + if (instance == null) { + instance = new EtcdConfigurator(); + } + } + } + return instance; + } + + @Override + public String getString(String key) { + return PROPERTIES.getString(key); + } + + @Override + public Long getLong(String key) { + return PROPERTIES.getLong(key); + } + + @Override + public List getList(String key) { + return PROPERTIES.getStrList(key); + } + + @Override + public T getObject(String key, Class clz) { + return PROPERTIES.getBean(key, clz); + } + + @Override + public boolean putConfig(String key, String content) { + return putConfig(key, content, DEFAULT_TIMEOUT); + } + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ + return false; + } + PROPERTIES.setProperty(key, content); + client.getKvClient().put(ByteString.copyFromUtf8(PROPERTIES_PATH), ByteString.copyFromUtf8(formatConfigStr(PROPERTIES))).sync(); + return true; + } + + + @Override + public String getType() { + return "etcd"; + } + +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java new file mode 100644 index 0000000000000000000000000000000000000000..b6c172b437f773c3184dd4b8bbc048a13e27d7d1 --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -0,0 +1,102 @@ +package com.jd.platform.jlog.etcd; + +import com.google.protobuf.ByteString; +import com.ibm.etcd.api.Event; +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.api.RangeResponse; +import com.ibm.etcd.client.EtcdClient; +import com.ibm.etcd.client.kv.KvClient; +import com.ibm.etcd.client.kv.WatchUpdate; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import com.jd.platform.jlog.core.ConfigChangeType; +import io.netty.util.concurrent.DefaultThreadFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES; +import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES_PATH; +import static com.jd.platform.jlog.etcd.EtcdConfigurator.ROOT; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName EtcdListener.java + * @Description TODO + * @createTime 2022年02月21日 23:34:00 + */ +public class EtcdListener implements ConfigChangeListener { + private KvClient.WatchIterator iterator; + + public EtcdListener() { + + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(ROOT)).asPrefix().start(); + getExecutorService().submit(() -> { + while (iterator.hasNext()){ + try { + WatchUpdate update = iterator.next(); + Event eve = update.getEvents().get(0); + KeyValue kv = eve.getKv(); + Event.EventType eveType = eve.getType(); + ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; + ConfigChangeEvent event = new ConfigChangeEvent(); + event.setKey(kv.getKey().toStringUtf8()).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + onChangeEvent(event); + }catch (RuntimeException e){ + e.printStackTrace(); + } + + } + }); + } + + + @Override + public void onShutDown() { + this.iterator.close(); + getExecutorService().shutdownNow(); + } + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + + RangeResponse rangeResponse = EtcdConfigurator.client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return; + } + String val = keyValues.get(0).getValue().toStringUtf8(); + JcProperties props = new JcProperties(); + if(StringUtil.isNotBlank(val)){ + try { + props.load(new StringReader(val)); + } catch (IOException e) { + e.printStackTrace(); + } + + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); + if(!diffKeys.isEmpty()){ + PROPERTIES = props; + for (String diffKey : diffKeys) { + LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", event.getKey(), diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey))); + } + ClientHandlerBuilder.refresh(); + } + } + } + +} diff --git a/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000000000000000000000000000000000000..32851cde74737f0883dabb2bea3ab7f4b2394998 --- /dev/null +++ b/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.etcd.EtcdConfigurationProvider \ No newline at end of file diff --git a/config/config-nacos/pom.xml b/config/config-nacos/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..911115ed75134e3af68a4197022776bda5497478 --- /dev/null +++ b/config/config-nacos/pom.xml @@ -0,0 +1,27 @@ + + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-nacos + config-nacos + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + com.alibaba.nacos + nacos-client + 1.4.2 + + + + diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java new file mode 100644 index 0000000000000000000000000000000000000000..b74fdabd7852cad703965cd4328a3fd7e64a5764 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -0,0 +1,145 @@ +package com.jd.platform.jlog.nacos; + +import java.io.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import com.alibaba.fastjson.JSON; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; + +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; +import static com.jd.platform.jlog.core.Constant.*; + + + +/** + * @author tangbohu + */ +public class NacosConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + private static volatile ConfigService configService; + + static volatile JcProperties PROPERTIES = new JcProperties(); + + static final String JLOG_GROUP = "JLOG_GROUP"; + + static final String DEFAULT_DATA_ID = "jLog.properties"; + + static NacosListener NACOSLISTENER = new NacosListener(); + + private static volatile NacosConfigurator instance; + + + public static NacosConfigurator getInstance() { + if (instance == null) { + synchronized (NacosConfigurator.class) { + if (instance == null) { + instance = new NacosConfigurator(); + } + } + } + return instance; + } + + + private NacosConfigurator() { + if (configService == null) { + try { + configService = NacosFactory.createConfigService(getConfigProperties()); + String config = configService.getConfig(DEFAULT_DATA_ID, JLOG_GROUP, DEFAULT_TIMEOUT); + LOGGER.info("从NaCos获取配置进行初始化 config = {}", config); + if (StringUtils.isNotBlank(config)) { + PROPERTIES.load(new StringReader(config)); + LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(PROPERTIES)); + configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, NACOSLISTENER); + } + } catch (NacosException | IOException e) { + throw new RuntimeException(e); + } + } + } + + + @Override + public String getString(String key) { + return PROPERTIES.getString(key); + } + + @Override + public Long getLong(String key) { + return PROPERTIES.getLong(key); + } + + @Override + public List getList(String key) { + return PROPERTIES.getStrList(key); + } + + @Override + public T getObject(String key, Class clz) { + return PROPERTIES.getBean(key, clz); + } + + + @Override + public boolean putConfig(String key, String content) { + return putConfig(key, content, DEFAULT_TIMEOUT); + } + + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + boolean result = false; + if(StringUtil.isEmpty(key)){ + return false; + } + try { + if (!PROPERTIES.isEmpty()) { + PROPERTIES.setProperty(key, content); + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, formatConfigStr(PROPERTIES)); + } else { + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, content); + } + } catch (NacosException ex) { + LOGGER.error(ex.getErrMsg()); + } + return result; + } + + + + private static Properties getConfigProperties() { + Properties properties = new Properties(); + String address = FILE_CONFIG.getString(SERVER_ADDR_KEY); + if (address != null) { + properties.setProperty(SERVER_ADDR_KEY, address); + } + + properties.setProperty(NAMESPACE_KEY, DEFAULT_NAMESPACE); + return properties; + } + + + + @Override + public String getType() { + return "nacos"; + } + + +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..f50b3f3603a2cedbc3cdcbb57056bd5cc3fa559b --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java @@ -0,0 +1,12 @@ +package com.jd.platform.jlog.nacos; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +public class NacosConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return NacosConfigurator.getInstance(); + } +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java new file mode 100644 index 0000000000000000000000000000000000000000..e1e4cc63fe6069f69840450e53a63d148a25199e --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -0,0 +1,76 @@ +package com.jd.platform.jlog.nacos; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.AbstractSharedListener; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.listener.Event; +import com.alibaba.nacos.api.naming.listener.EventListener; +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.Properties; +import java.util.Set; + + +import static com.jd.platform.jlog.core.Constant.*; +import static com.jd.platform.jlog.nacos.NacosConfigurator.*; + + +/** + * @author tangbohu + */ +public class NacosListener extends AbstractSharedListener implements ConfigChangeListener, EventListener { + + + private static final Logger LOGGER = LoggerFactory.getLogger(NacosListener.class); + + + public NacosListener() {} + + @Override + public void innerReceive(String dataId, String group, String configInfo) { + LOGGER.debug("configInfo:{}", configInfo); + ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo).setNamespace(group); + this.onProcessEvent(event); + } + + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + LOGGER.debug("通用[配置]变更事件 event:{}",event); + JcProperties props = new JcProperties(); + if (StringUtils.isNotBlank(event.getNewValue())) { + try { + props.load(new StringReader(event.getNewValue())); + } catch (IOException e) { + return; + } + } + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); + if(!diffKeys.isEmpty()){ + PROPERTIES = props; + for (String diffKey : diffKeys) { + LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", DEFAULT_DATA_ID, diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey))); + } + ClientHandlerBuilder.refresh(); + } + } + + + @Override + public void onEvent(Event event) { + LOGGER.info("通用[服务]事件 event:{}",event); + } + +} diff --git a/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000000000000000000000000000000000000..6d982a37ac9992888c34b2be9f6a79a1e816e864 --- /dev/null +++ b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.nacos.NacosConfiguratorProvider \ No newline at end of file diff --git a/config/config-nacos/src/test/NacosTest.java b/config/config-nacos/src/test/NacosTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a67bceddddd7d681765c34a27a79704645987b5a --- /dev/null +++ b/config/config-nacos/src/test/NacosTest.java @@ -0,0 +1,68 @@ +package com.jd.platform.jlog.nacos; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingFactory; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.listener.NamingEvent; + +import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; +import static com.jd.platform.jlog.nacos.NacosConstant.JLOG_GROUP; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosTest.java + * @Description TODO + * @createTime 2022年02月22日 22:04:00 + */ +public class NacosTest { + + public static void main(String[] args) throws NacosException, InterruptedException { + String serverAddr = "101.42.242.201:8848"; + String dataId = DEFAULT_DATA_ID; + String group = JLOG_GROUP; + ConfigService configService = NacosFactory.createConfigService(serverAddr); + + NamingService naming = NamingFactory.createNamingService(serverAddr); + naming.registerInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1"); + Thread.sleep(3000); + NacosListener server = new NacosListener(); + naming.subscribe("nacos.test1", JLOG_GROUP,server); + String content = configService.getConfig(dataId, group, 2000L); + System.out.println("content: "+content); + NacosListener nL = new NacosListener(); + configService.addListener(dataId, group, nL); + + System.out.println("新增完成"); + Thread.sleep(20000); + System.out.println("移除开始"); + configService.removeListener(dataId, group, nL); + System.out.println("移除结束"); + + naming.deregisterInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1"); + Thread.sleep(5000); + + System.out.println("睡眠结束"); + + /* System.out.println("移除开始222"); + configService.removeListener(dataId, group, new Listener() { + @Override + public Executor getExecutor() { + return Executors.newSingleThreadExecutor(); + } + + @Override + public void receiveConfigInfo(String s) { + System.out.println("#### 移除开始222"); + } + });*/ + + Thread.sleep(90000); + System.out.println("---------------------------睡眠结束"); + + } + + +} diff --git a/config/config-zk/pom.xml b/config/config-zk/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..93e9f4bc314242eaba8c3dc7bf83d64a0081785a --- /dev/null +++ b/config/config-zk/pom.xml @@ -0,0 +1,41 @@ + + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-zk + config-zk + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + org.apache.curator + curator-recipes + 4.2.0 + + + + diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java new file mode 100644 index 0000000000000000000000000000000000000000..ebaa734d38665dc07c799dbe0de4771038e3f2be --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -0,0 +1,119 @@ +package com.jd.platform.jlog.zk; + +import java.io.ByteArrayInputStream; +import java.util.List; +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.*; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte; +import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; +import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; + + +/** + * @author tangbohu + */ +public class ZkConfigurator implements Configurator { + + private final static Logger LOGGER = LoggerFactory.getLogger(ZkConfigurator.class); + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + static volatile CuratorFramework zkClient; + + static volatile JcProperties PROPERTIES = new JcProperties(); + + private static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; + + + public ZkConfigurator() throws Exception { + if (zkClient == null) { + synchronized (ZkConfigurator.class) { + zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getString(SERVER_ADDR_KEY)) + // 连接超时时间 + .sessionTimeoutMs(6000) + // 会话超时时间 + .connectionTimeoutMs(2000) + .namespace(DEFAULT_NAMESPACE) + // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + zkClient.start(); + } + + loadZkData(); + new ZkListener(DEFAULT_CONFIG_PATH); + LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(PROPERTIES)); + } + } + + + @Override + public String getString(String key) { + return PROPERTIES.getString(key); + } + + @Override + public Long getLong(String key) { + return PROPERTIES.getLong(key); + } + + @Override + public List getList(String key) { + return PROPERTIES.getStrList(key); + } + + @Override + public T getObject(String key, Class clz) { + return PROPERTIES.getBean(key, clz); + } + + + @Override + public boolean putConfig(String key, String content) { + return putConfig(key, content, DEFAULT_TIMEOUT); + } + + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ + return false; + } + PROPERTIES.setProperty(key, content); + try { + zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigByte(PROPERTIES)); + } catch (Exception e) { + return false; + } + return true; + } + + + + @Override + public String getType() { + return "zk"; + } + + + + static void loadZkData() throws Exception { + + byte[] bt = zkClient.getData().forPath(DEFAULT_CONFIG_PATH); + if (bt != null && bt.length > 0){ + ByteArrayInputStream bArray = new ByteArrayInputStream(bt); + PROPERTIES.load(bArray); + } + LOGGER.info("# loadZkData # PROPERTIES:{}",JSON.toJSONString(PROPERTIES)); + } + +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..e2355e5e319886e613e77d54478d2911a479bf1e --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java @@ -0,0 +1,20 @@ +package com.jd.platform.jlog.zk; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + + +/** + * @author tangbohu + */ +public class ZkConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + try { + return new ZkConfigurator(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java new file mode 100644 index 0000000000000000000000000000000000000000..e41b07680c28164df3b1d8a9bf59cd398d2b69d9 --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -0,0 +1,70 @@ +package com.jd.platform.jlog.zk; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import com.jd.platform.jlog.core.ConfigChangeType; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static com.jd.platform.jlog.zk.ZkConfigurator.*; + + +/** + * @author didi + */ +public class ZkListener implements ConfigChangeListener { + + private final static Logger LOGGER = LoggerFactory.getLogger(ZkListener.class); + + private NodeCache cache; + + private String path; + + public ZkListener(String path) { + this.path = path; + cache = new NodeCache(zkClient, path); + try { + cache.start(true); + cache.getListenable().addListener(() -> onChangeEvent(null)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + LOGGER.info("ZK数据变更-当前监听器关注的path:{} PROPERTIES:{}", path, JSON.toJSONString(PROPERTIES)); + Properties props = new Properties(); + props.putAll(PROPERTIES); + try { + LOGGER.debug("ZK数据变更,旧Properties:{}", JSON.toJSONString(props)); + loadZkData(); + LOGGER.debug("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES)); + } catch (Exception e) { + e.printStackTrace(); + } + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); + if(!diffKeys.isEmpty()){ + for (String diffKey : diffKeys) { + LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey, + new ConfigChangeEvent(diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey)))); + } + ClientHandlerBuilder.refresh(); + } + } + +} diff --git a/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000000000000000000000000000000000000..f8a41f9bd0a016a986ee9bd52f33d4399cd6d551 --- /dev/null +++ b/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.zk.ZkConfiguratorProvider \ No newline at end of file diff --git a/config/pom.xml b/config/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..e9885c4a85282912405bbf89807a589666d29eac --- /dev/null +++ b/config/pom.xml @@ -0,0 +1,36 @@ + + + + + JLog + com.jd.platfrom.jlog + 1.4-SNAPSHOT + + 4.0.0 + config + pom + config + + config-core + config-etcd + config-apollo + config-nacos + config-zk + + diff --git a/clientdemo/pom.xml b/example/pom.xml similarity index 54% rename from clientdemo/pom.xml rename to example/pom.xml index 052e0c41e4f90bece80cbc4f2af9443f2cb286d7..b177a8e1bd2e30cdac701d1cee4a814aedcfb51a 100644 --- a/clientdemo/pom.xml +++ b/example/pom.xml @@ -10,10 +10,10 @@ com.jd.platfrom.jlog - clientdemo + example 0.0.1-SNAPSHOT - clientdemo - Demo project for using JLog + example + example for using JLog 1.8 @@ -23,42 +23,71 @@ org.springframework.boot spring-boot-starter-web - + - + com.jd.platfrom.jlog client 1.4-SNAPSHOT - org.springframework.boot - spring-boot-starter-test - test + spring-boot-configuration-processor + true - - com.jd.platfrom.jlog - clientlog4j2 - 1.4-SNAPSHOT + org.springframework.boot + spring-boot-test + 2.5.5 + test junit junit - 4.12 + 4.13.2 test - junit - junit - 4.12 + org.springframework + spring-test + 5.3.10 test + + org.projectlombok + lombok + 1.18.20 + test + + + diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java similarity index 63% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java index fb1e00d689580ddfcb593245fff50052e638b5f6..fec83ce64b3657418c5055fa1e3deb651d65479b 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java @@ -11,9 +11,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; * @date 2021-12-27 */ @SpringBootApplication -public class ClientDemoApplication { +public class ExampleApplication { public static void main(String[] args) { - SpringApplication.run(ClientDemoApplication.class, args); + try { + SpringApplication.run(ExampleApplication.class, args); + }catch (Exception e){ + e.printStackTrace(); + } } } diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java new file mode 100644 index 0000000000000000000000000000000000000000..96c4de78c8ab23833b01f6d414ec39ecfd47f9de --- /dev/null +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -0,0 +1,115 @@ +package com.jd.platform.jlog.clientdemo.appender; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxy; +import ch.qos.logback.core.AppenderBase; +import com.jd.platform.jlog.client.log.LogExceptionStackTrace; +import com.jd.platform.jlog.client.tracerholder.TracerHolder; +import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.common.model.RunLogMessage; +import com.jd.platform.jlog.core.ClientHandler; +import org.slf4j.helpers.MessageFormatter; + + +/** + * className：TracerLog4JAppender + * + * @author wuweifeng + * @version 1.0.0 + */ +public class TracerLogbackAppender extends AppenderBase { + /** + * 日志的堆栈级别，如直接在类里使用logger.info，则该值为0. 如调用另一个日志类如CommonLogger打印，则该值为1 + */ + private int loggerStage; + + @Override + protected void append(ILoggingEvent iLoggingEvent) { + try { + long tracerId = TracerHolder.getTracerId(); + if (0L == tracerId) { + return; + } + RunLogMessage logMessage = getLogMessage(iLoggingEvent); + UdpSender.offerLogger(logMessage); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 转化为对象 + */ + private RunLogMessage getLogMessage(ILoggingEvent loggingEvent) { + RunLogMessage logMessage = new RunLogMessage(); + //设置链路唯一id + logMessage.setTracerId(TracerHolder.getTracerId()); + logMessage.setClassName(loggingEvent.getLoggerName()); + logMessage.setThreadName(loggingEvent.getThreadName()); + + StackTraceElement stackTraceElement = loggingEvent.getCallerData()[loggerStage]; + + String method = stackTraceElement.getMethodName(); + String line = String.valueOf(stackTraceElement.getLineNumber()); + logMessage.setMethodName(method + "(" + stackTraceElement.getFileName() + ":" + line + ")"); + logMessage.setLogLevel(loggingEvent.getLevel().toString()); + + logMessage.setCreateTime(loggingEvent.getTimeStamp()); + + String formattedMessage = getMessage(loggingEvent); + logMessage.setContent(formattedMessage); + ClientHandler.Outcome out = ClientHandler.processLog(formattedMessage); + logMessage.setTagMap(out.getMap()); + return logMessage; + } + + /** + * 日志正文信息 + */ + private String getMessage(ILoggingEvent logEvent) { + if (logEvent.getLevel() == Level.ERROR) { + if (logEvent.getThrowableProxy() != null) { + ThrowableProxy throwableProxy = (ThrowableProxy) logEvent.getThrowableProxy(); + String[] args = new String[]{logEvent.getFormattedMessage() + "\n" + LogExceptionStackTrace.erroStackTrace(throwableProxy.getThrowable()).toString()}; + return packageMessage("{}", args); + } else { + Object[] args = logEvent.getArgumentArray(); + if (args != null) { + for (int i = 0; i < args.length; i++) { + if (args[i] instanceof Throwable) { + args[i] = LogExceptionStackTrace.erroStackTrace(args[i]); + } + } + return packageMessage(logEvent.getMessage(), args); + } + } + } + return logEvent.getFormattedMessage(); + } + + private String packageMessage(String message, Object[] args) { + if (message != null && message.contains("{}")) { + return MessageFormatter.arrayFormat(message, args).getMessage(); + } + return packageMsg(message, args); + } + + private String packageMsg(String message, Object[] args) { + StringBuilder builder = new StringBuilder(128); + builder.append(message); + for (Object arg : args) { + builder.append("\n").append(arg); + } + return builder.toString(); + } + + + public int getLoggerStage() { + return loggerStage; + } + + public void setLoggerStage(int loggerStage) { + this.loggerStage = loggerStage; + } +} diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java similarity index 52% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java index 5b4e4237a18afda1967a265a911400d00ab73789..00a7fe81bfe54ddedb5aa107978c03d985106caf 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java @@ -1,12 +1,14 @@ package com.jd.platform.jlog.clientdemo.config; import com.jd.platform.jlog.client.TracerClientStarter; -import com.jd.platform.jlog.client.filter.UserFilter; -import org.springframework.beans.factory.annotation.Value; +import com.jd.platform.jlog.client.filter.HttpFilter; +import com.jd.platform.jlog.common.handler.TagConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - +import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; /** @@ -16,24 +18,37 @@ import javax.annotation.PostConstruct; * @version 1.0 * @date 2021-12-27 */ -@Configuration +@Component +@ConfigurationProperties() public class DemoConfig { - @Value("${config.server}") - private String etcdServer; + private Logger logger = LoggerFactory.getLogger(getClass()); + + private TagConfig tagConfig ; + + public TagConfig getTagConfig() { + return tagConfig; + } + + public void setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; + } @PostConstruct - public void begin() { + public void begin() throws Exception { + TracerClientStarter tracerClientStarter = new TracerClientStarter.Builder() .setAppName("demo") - .setEtcdServer(etcdServer).build(); + .setTagConfig(tagConfig) + .build(); + logger.info("初始化tagConfig: {}",tagConfig); tracerClientStarter.startPipeline(); } @Bean public FilterRegistrationBean urlFilter() { FilterRegistrationBean registration = new FilterRegistrationBean(); - UserFilter userFilter = new UserFilter(); + HttpFilter userFilter = new HttpFilter(); registration.setFilter(userFilter); registration.addUrlPatterns("/*"); diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java similarity index 30% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 60929eaf0dbd1c7a7bf2d8303ca01fca303d21c0..0a3b3857aeaf5a9e1ba43814881edde886626357 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -1,19 +1,36 @@ package com.jd.platform.jlog.clientdemo.web; + import com.jd.platform.jlog.common.model.TracerBean; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; /** * @author shenkaiwen5 * @version 1.0 * @date 2021-12-27 */ +@Component +//@ConfigurationProperties() @RestController public class TestController { + private TagConfig tagConfig ; + + public TagConfig getTagConfig() { + return tagConfig; + } + + public void setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; + } + /** * do nothing * just as an adapter for this project common log helper @@ -22,13 +39,40 @@ public class TestController { private static Logger RequestLog = LoggerFactory.getLogger("RequestLog"); @RequestMapping("/index") - public Object index() { + public Object index() { TracerBean tracerBean = new TracerBean(); tracerBean.setTracerId("11111"); + Configurator configurator = ConfiguratorFactory.getInstance(); + try{ + configurator.putConfig("/test","val1"); + }catch (Exception e){ + e.printStackTrace(); + } + + String val = configurator.getString("/test"); + System.out.println("val ===> "+val); RequestLog.info("哈哈哈哈哈哈"); return tracerBean; } + @RequestMapping("/log") + public Object log() { + RequestLog.info("|errno=val3||node=val4||这是随便的log|"); + return 1; + } + + + @PostMapping(value = "/test", consumes = MediaType.APPLICATION_JSON_VALUE) + public Object test(@RequestParam Integer uid, @RequestParam Integer newKey,@RequestBody TestReq req) { + String config = ConfiguratorFactory.getInstance().getString("reqTags"); + // System.out.println("tagConfig ===> " + tagConfig.toString()); + RequestLog.info("|errno=val3||node=val4||这是随便的log|"); + + return 1; + } + + + } diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java new file mode 100644 index 0000000000000000000000000000000000000000..917c1b399369ebad2b51b0687f1213430dd6cdd7 --- /dev/null +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.clientdemo.web; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName TestReq.java + * @Description TODO + * @createTime 2022年02月17日 16:40:00 + */ +public class TestReq { + + private Integer id; + + private String name; +} diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..19b36d20b1fe750d5a73596aaa3efb7176c55f8c --- /dev/null +++ b/example/src/main/resources/application.properties @@ -0,0 +1,6 @@ +server.port=8085 +serverAddr=101.42.242.201:2379 + +apollo.meta=http://127.0.0.1:8080 +apollo.config-service=http://127.0.0.1:8080 +app.id=order \ No newline at end of file diff --git a/example/src/main/resources/bakapplication.yml b/example/src/main/resources/bakapplication.yml new file mode 100644 index 0000000000000000000000000000000000000000..2bb1bcb9c978226a2b11960f182c1eccdd65b2c1 --- /dev/null +++ b/example/src/main/resources/bakapplication.yml @@ -0,0 +1,24 @@ +workers: [1,2,3] +compress: 68 +threshold: 10000 + +serverAddr: 101.42.242.201:2181 +server: + port: 8085 + +test: true + + +tag-config: + reqTags: + - uid + - url + - newK + logTags: + - node + - bizType + respTags: + - errno + delimiter: "|" + join: "=" + diff --git a/example/src/main/resources/bakjLog.properties b/example/src/main/resources/bakjLog.properties new file mode 100644 index 0000000000000000000000000000000000000000..697d62781d37f9c0ae3b33f8a1337a501581612f --- /dev/null +++ b/example/src/main/resources/bakjLog.properties @@ -0,0 +1,18 @@ +server.port=8085 + +tag-config.reqTags[0]=uid +tag-config.reqTags[1]=url + +tag-config.logTags[0]=node +tag-config.logTags[1]=bizType + +tag-config.respTags[0]=errno +tag-config.respTags[1]=msg +tag-config.delimiter=| +tag-config.join== +tag-config.extract=41 + +compress=68 +threshold=10000 + +workers=[1,2,3] diff --git a/example/src/main/resources/logback-spring.xml b/example/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000000000000000000000000000000000..8b56e11c64a693cc2c1b6bcb178ea11c0b17fb25 --- /dev/null +++ b/example/src/main/resources/logback-spring.xml @@ -0,0 +1,62 @@ + + + logs + + + + + + + + + + + + ${log.colorPattern} + + + + + + + + + + ${log.path}/info/info.%d{yyyy-MM-dd}.log + ${log.maxHistory} + + + ${log.pattern} + + + INFO + ACCEPT + DENY + + + + + + ${log.path}/error/error.%d{yyyy-MM-dd}.log + + + ${log.pattern} + + + ERROR + ACCEPT + DENY + + + + + + + + + + + + + + diff --git a/example/src/test/java/com/jd/platform/jlog/test/Common.java b/example/src/test/java/com/jd/platform/jlog/test/Common.java new file mode 100644 index 0000000000000000000000000000000000000000..b6034fb01788e7e7e8e891def7e8147e91222b4a --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/Common.java @@ -0,0 +1,67 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.core.Configurator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.List; +import java.util.Random; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Common.java + * @Description TODO + * @createTime 2022年03月01日 07:36:00 + */ +public class Common { + + private final static Logger LOGGER = LoggerFactory.getLogger(Common.class); + + + public static void getTest(Configurator configurator){ + + LOGGER.info("配置器类型：{}", configurator.getType()); + String addr = configurator.getString("serverAddr"); + LOGGER.info("配置器get addr：{}", addr); + TagConfig tagConfig = configurator.getObject("tag-config", TagConfig.class); + LOGGER.info("配置器get tagConfig：{}", tagConfig.toString()); + List workers = configurator.getList("workers"); + LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); + } + + + public static void modifyFile(String path)throws Exception{ + + String temp; + File file = new File(path); + FileInputStream fis = new FileInputStream(file); + InputStreamReader isr = new InputStreamReader(fis); + BufferedReader br = new BufferedReader(isr); + StringBuffer buf = new StringBuffer(); + + int id = new Random().nextInt(1000); + int num = 0; + // 保存该行前面的内容 + while ((temp = br.readLine()) != null) { + if(num == 0){ + buf = buf.append("testKey: ").append(id); + }else{ + buf = buf.append(temp); + } + num++; + buf = buf.append(System.getProperty("line.separator")); + } + + br.close(); + FileOutputStream fos = new FileOutputStream(file); + PrintWriter pw = new PrintWriter(fos); + pw.write(buf.toString().toCharArray()); + pw.flush(); + pw.close(); + } + +} diff --git a/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4a5b575bd2d6ad3091238a1ea767edff8482d1ec --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java @@ -0,0 +1,85 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName EtcdConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月03日 07:35:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class EtcdConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(EtcdConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + // @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getList("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.1.1.0"; + if(workers.contains(myIp)){ + // do nothing + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getList("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getString("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); + configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(5000); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + Thread.sleep(5000); + // LOGGER.info("移除监听器后:修改配置testKey = {}",i2); + // configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + LOGGER.info("第二次添加监听器"); + Thread.sleep(22000); + + } +} diff --git a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..e3eca92b524eb4580530cba6525c6def267ea7f8 --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -0,0 +1,69 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; +import java.util.Map; +import java.util.Properties; + +import static com.jd.platform.jlog.test.Common.getTest; +import static com.jd.platform.jlog.test.Common.modifyFile; + + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileConfiguratorTest.java + * @Description TODO + * @createTime 2022年02月28日 19:45:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class FileConfiguratorTest { + + + private final static Logger LOGGER = LoggerFactory.getLogger(FileConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + + + @Test + public void testAddConfigListener() throws Exception { + String path = "/Users/didi/Desktop/jlog/example/target/classes/bakapplication.yml"; + Properties props = new Properties(); + FileInputStream fis = new FileInputStream(new File(path)); + if (path.contains("yml")) { + props.putAll(new Yaml().loadAs(fis, Map.class)); + } else { + props.load(fis); + } + LOGGER.info("读取文件：{} 最新配置:{}", path, JSON.toJSONString(props)); + modifyFile(path); + LOGGER.info("修改文件完毕 准备触发监听器"); + Thread.sleep(10000); + } + + + +} diff --git a/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..38c77c9c64747618f700a7f484e03afc928e4530 --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java @@ -0,0 +1,81 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月01日 07:35:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class NacosConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(NacosConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getList("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.1.1.0"; + if(workers.contains(myIp)){ + // do nothing + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getList("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getString("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + // configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(1000); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + + Thread.sleep(35000); + } +} diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java b/example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java similarity index 87% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java rename to example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java index e2a834341d761ffaad32a163cd091ddbe72afb7f..0ef5a231e35cef2cd0b2edccd80faf67e791d1c4 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java @@ -2,15 +2,11 @@ package com.jd.platform.jlog.test; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.clientdemo.ExampleApplication; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IpUtils; import com.jd.platform.jlog.common.utils.ZstdUtils; import org.apache.tomcat.util.codec.binary.Base64; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; import java.nio.charset.StandardCharsets; import java.util.*; @@ -22,11 +18,11 @@ import java.util.*; * @version 1.0 * @date 2021-12-27 */ -@SpringBootTest(classes = ClientDemoApplication.class) -@RunWith(SpringRunner.class) +//@SpringBootTest(classes = ExampleApplication.class) +//@RunWith(SpringRunner.class) public class TracerPacketTest { - @Test + //@Test public void testSendUdp() { TracerBean tracerBean = new TracerBean(); List> tracerObject = new ArrayList<>(); diff --git a/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..26582a433bec7c1f4894ab1524fee639f6ca4267 --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java @@ -0,0 +1,79 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; + +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ZKConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月01日 07:35:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class ZKConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(ZKConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getList("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.0"; + if(workers.contains(myIp)){ + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getList("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getString("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); + // configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(1000); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + Thread.sleep(1000); + } +} diff --git a/pom.xml b/pom.xml index 57bd6e24d557b98279985469e2dc7852912b2862..4c92e3e9002abfa8edba6caca244cdebebefa8ef 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ clientlog4j2 clientlogback Dashboard - clientdemo + example + config diff --git a/worker/pom.xml b/worker/pom.xml index 771b8f6fe310d4cc887583ceb163927f399e2afe..8e02fe62289fe29f13683f7e9b1e5da2df4e79b7 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -22,6 +22,11 @@ org.springframework.boot spring-boot-starter-web + + com.jd.platfrom.jlog + config-zk + 1.4-SNAPSHOT + org.springframework.boot @@ -43,6 +48,12 @@ disruptor 3.4.1 + + + org.springframework.boot + spring-boot-configuration-processor + true + diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java b/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java index 7c76876b3c416fb670e3651db570139ab5a84cf8..08ff060aec5ee2db88fa6fe0fbbdc96aa4dcbf59 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java @@ -1,6 +1,6 @@ package com.jd.platform.jlog.worker; -import com.jd.platform.jlog.worker.config.EtcdStarter; +import com.jd.platform.jlog.worker.config.CenterStarter; import com.jd.platform.jlog.worker.store.TracerLogToDbStore; import com.jd.platform.jlog.worker.store.TracerModelToDbStore; import com.jd.platform.jlog.worker.udp.UdpServer; @@ -42,7 +42,7 @@ public class Starter { * etcd启动器 */ @Resource - private EtcdStarter etcdStarter; + private CenterStarter centerStarter; @PostConstruct public void start() { @@ -58,7 +58,7 @@ public class Starter { tracerLogToDbStore.beginIntoDb(); //上报自己ip到配置中心 - etcdStarter.uploadSelfInfo(); + centerStarter.uploadSelfInfo(); } } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java similarity index 75% rename from worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java rename to worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java index 67aac4a397bf6fd7bfed00c0fabe838c0c018af9..9c234f4cd76880b358b366a11a8b5eef88a4a472 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java @@ -1,12 +1,16 @@ package com.jd.platform.jlog.worker.config; -import com.jd.platform.jlog.common.config.IConfigCenter; +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.constant.Constant; import com.jd.platform.jlog.common.utils.IpUtils; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.Resource; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -19,7 +23,7 @@ import java.util.concurrent.TimeUnit; * @date 2021-08-12 */ @Component -public class EtcdStarter { +public class CenterStarter { /** * 该worker为哪个app服务 */ @@ -35,23 +39,23 @@ public class EtcdStarter { */ @Value("${config.mdc}") private String mdc; - /** - * configCenter - */ - @Resource - private IConfigCenter configCenter; + /** * 上报自己的ip到配置中心 */ public void uploadSelfInfo() { //开启上传worker信息 + Configurator config = ConfiguratorFactory.getInstance(); ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); scheduledExecutorService.scheduleAtFixedRate(() -> { try { - configCenter.putAndGrant(buildKey(), buildValue(), 8); - configCenter.putAndGrant(buildSecondKey(), buildValue(), 8); + List list = config.getList("workers"); + if(!list.contains(buildKey())){ + list.add(buildValue()); + config.putConfig("workers", JSON.toJSONString(list)); + } } catch (Exception e) { //do nothing e.printStackTrace(); @@ -68,14 +72,6 @@ public class EtcdStarter { return Constant.WORKER_PATH + workerPath + "/" + hostName; } - /** - * 在配置中心对应机房存放的key - */ - private String buildSecondKey() { - String hostName = IpUtils.getHostName(); - return Constant.WORKER_PATH + workerPath + "/" + mdc + "/" + hostName; - } - /** * 在配置中心存放的value */ diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java deleted file mode 100644 index 4195e9a94b3ffebf7b170aa0238cc0a9b13e1f6b..0000000000000000000000000000000000000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.jlog.worker.config; - -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -/** - * EtcdConfig - * @author wuweifeng wrote on 2019-12-06 - * @version 1.0 - */ -@Configuration -public class EtcdConfig { - @Value("${config.server}") - private String etcdServer; - - private Logger logger = LoggerFactory.getLogger(getClass()); - - @Bean - public IConfigCenter client() { - logger.info("etcd address : " + etcdServer); - //连接多个时，逗号分隔 - return JdEtcdBuilder.build(etcdServer); - } - -} diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java deleted file mode 100644 index 064182ae287c1d408849c26358ccc8994cff832a..0000000000000000000000000000000000000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.jd.platform.jlog.worker.config; - -/** - * 配置中心接口 - * @author wuweifeng - * @version 1.0 - * @date 2021-08-13 - */ -public interface IConfig { -} diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java b/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java index 716799557bddc50fedd503db0d98df5fb063899b..98ebaf79c91c30de913afb38acee24a3d1fc5e47 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.worker.controller; -import cn.hutool.core.date.DateUtil; import com.jd.platform.jlog.worker.db.Db; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -32,12 +31,12 @@ public class UserController { Map map0 = new HashMap<>(); map0.put("clientType", 1); map0.put("pin", "abcd"); - map0.put("createTime", DateUtil.formatDateTime(new Date())); + // map0.put("createTime", DateUtil.formatDateTime(new Date())); Map map1 = new HashMap<>(); map1.put("clientType", 1); map1.put("pin", "abcd"); - map1.put("createTime", DateUtil.formatDateTime(new Date())); + // map1.put("createTime", DateUtil.formatDateTime(new Date())); datas.add(map0); datas.add(map1); diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java index 7469555bcde27020277d22865a8d7b11e2f358f3..167bcc0e17dfc65250385d0c4c0d1225494a51fa 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.worker.disruptor; -import cn.hutool.core.date.DateUtil; import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.model.TracerData; @@ -105,8 +104,8 @@ public class TracerConsumer implements WorkHandler { map.put("threadName", runLogMessage.getThreadName()); map.put("methodName", runLogMessage.getMethodName()); map.put("logLevel", runLogMessage.getLogLevel()); - map.put("createTime", DateUtil.formatDateTime(new Date(runLogMessage.getCreateTime()))); map.put("content", runLogMessage.getContent()); + map.putAll(runLogMessage.getTagMap()); tracerLogToDbStore.offer(map); } @@ -116,9 +115,12 @@ public class TracerConsumer implements WorkHandler { * 处理filter里处理的出入参 */ private void dealFilterModel(TracerBean tracerBean) { + List> mapList = tracerBean.getTracerObject(); Map requestMap = mapList.get(0); + Map map = new HashMap<>(requestMap); + long tracerId = requestMap.get("tracerId") == null ? 0 : Long.valueOf(requestMap.get("tracerId").toString()); //filter的出入参 Map responseMap = mapList.get(mapList.size() - 1); @@ -128,59 +130,11 @@ public class TracerConsumer implements WorkHandler { responseBytes = (byte[]) responseMap.get("response"); } - Map map = new HashMap<>(); - //jsf的是用户自己设置的request入参，http的是从httpRequest读取的 - if (requestMap.get("wholeRequest") == null) { - map.put("requestContent", FastJsonUtils.collectToString(requestMap)); - } else { - map.put("requestContent", requestMap.get("wholeRequest")); - } - - //此处做了一个base64编码，否则原编码直接进去，取出来后是String，直接getBytes后无法用Zstd解压 map.put("responseContent", responseBytes); - map.put("createTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); map.put("costTime", tracerBean.getCostTime()); - - map.put("tracerId", tracerId); - - String pin = requestMap.get("pin") == null ? "" : requestMap.get("pin").toString(); - map.put("pin", pin); - - String uri = requestMap.get("uri") == null ? "" : requestMap.get("uri").toString(); - map.put("uri", uri); - - //appName - String appName = requestMap.get("appName") == null ? "" : requestMap.get("appName").toString(); - map.put("appName", appName); - - String openudid = requestMap.get("openudid") == null ? "" : requestMap.get("openudid").toString(); - - if (StringUtil.isNullOrEmpty(openudid)) { - String uuid = requestMap.get("uuid") == null ? "" : requestMap.get("uuid").toString(); - map.put("uuid", uuid); - } else { - map.put("uuid", openudid); - } - - String client = requestMap.get("client") == null ? "" : requestMap.get("client").toString(); - int clientType = 0; - if ("apple".equals(client)) { - clientType = 2; - } else if ("android".equals(client)) { - clientType = 1; - } - map.put("clientType", clientType); - String clientVersion = requestMap.get("clientVersion") == null ? "" : requestMap.get("clientVersion").toString(); - map.put("clientVersion", clientVersion); - - String userIp = requestMap.get("ip") == null ? "" : requestMap.get("ip").toString(); - map.put("userIp", userIp); - String serverIp = requestMap.get("serverIp") == null ? "" : requestMap.get("serverIp").toString(); - map.put("serverIp", serverIp); - - map.put("intoDbTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); - + responseMap.remove("response"); + map.putAll(responseMap); tracerModelToDbStore.offer(map); } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java b/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java deleted file mode 100644 index b359adef09c623c1008480451fbc1a06c1e8adf6..0000000000000000000000000000000000000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.jd.platform.jlog.worker.entity; - - -import java.io.Serializable; - -/** - * 入库、检索时用的对象 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-08-20 - */ -public class TracerModel implements Serializable { - /** - * tracerId - */ - private long tracerId; - /** - * 请求的入参 - */ - private String requestContent; - /** - * 响应的出参 - */ - private String responseContent; - /** - * 日志请求时间（数据库里存的是DateTime，2021-08-24 19:47:30.0） - */ - private long createTime; - /** - * 请求耗时（即响应时间戳减去请求时间戳） - */ - private int costTime; - /** - * 用户pin - */ - private String pin; - /** - * 用户uuid - */ - private String uuid; - /** - * Android=1、ios=2 - */ - private int clientType; - /** - * 客户端版本号，9.3.6 - */ - private String clientVersion; - /** - * 用户ip - */ - private String userIp; - /** - * 服务端ip - */ - private String serverIp; - /** - * 入库时间 - */ - private long intoDbTime; - - - public long getIntoDbTime() { - return intoDbTime; - } - - public void setIntoDbTime(long intoDbTime) { - this.intoDbTime = intoDbTime; - } - - public String getRequestContent() { - return requestContent; - } - - public void setRequestContent(String requestContent) { - this.requestContent = requestContent; - } - - public String getResponseContent() { - return responseContent; - } - - public void setResponseContent(String responseContent) { - this.responseContent = responseContent; - } - - public long getTracerId() { - return tracerId; - } - - public void setTracerId(long tracerId) { - this.tracerId = tracerId; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - public int getCostTime() { - return costTime; - } - - public void setCostTime(int costTime) { - this.costTime = costTime; - } - - public String getPin() { - return pin; - } - - public void setPin(String pin) { - this.pin = pin; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public int getClientType() { - return clientType; - } - - public void setClientType(int clientType) { - this.clientType = clientType; - } - - public String getClientVersion() { - return clientVersion; - } - - public void setClientVersion(String clientVersion) { - this.clientVersion = clientVersion; - } - - - public String getUserIp() { - return userIp; - } - - public void setUserIp(String userIp) { - this.userIp = userIp; - } - - public String getServerIp() { - return serverIp; - } - - public void setServerIp(String serverIp) { - this.serverIp = serverIp; - } -} diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java index e226b6d6bde2b8eb19c923716fbdca6b0a5e3a75..fa8b5fc1efa447918971847923292c8021cbf117 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.worker.store; -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.collect.Queues; + import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.AsyncWorker; import com.jd.platform.jlog.worker.db.Db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,8 +94,8 @@ public class TracerLogToDbStore { try { List> tempModels = new ArrayList<>(); //每1s入库一次 - Queues.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempModels)) { + AsyncWorker.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS); + if (tempModels.size() == 0) { continue; } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java index 4ad9d8f5193963733971632e27c07bc38ac1fdb4..8a54550fabf5dece5a05e689de45eb4cb191a9af 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.worker.store; -import cn.hutool.core.collection.CollectionUtil; import com.google.common.collect.Queues; import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.worker.db.Db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 019ac4818a2cef6bb4a16782217c5511c84b0867..924c7ae31707ccf0c3ce3c310d774b814ec85b6f 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -6,10 +6,8 @@ queue: maxSize: ${queueSize:16384} preDbSize: ${preDbSize:10000} #etcd的地址，如有多个用逗号分隔 -config: - mdc: ${mdc:default} - server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ - workerPath: ${workerPath:default} #该worker放到哪个path下，譬如放/app1下，则该worker只能被app1使用，不会为其他client提供服务 +serverAddr: 101.42.242.201:2181 + server: port: 8080 #ck信息，自行修改 diff --git a/worker/src/main/resources/logback-spring.xml b/worker/src/main/resources/logback-spring.xml index 557bfde559b2441b8711f9df1d58c4f9a99ca898..31d2e3426adddbd35c4f5992a7e8c3d951c5b1bb 100644 --- a/worker/src/main/resources/logback-spring.xml +++ b/worker/src/main/resources/logback-spring.xml @@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息， --> - +