From 6a51bd2a670c94d9866c3dc970bea3a1251d0978 Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Thu, 6 Jan 2022 21:44:16 +0800 Subject: [PATCH 01/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clientdemo/clientdemo.iml | 9 +- ...ub_spotbugs_spotbugs_annotations_3_1_9.xml | 13 ++ .../Maven__io_netty_netty_3_10_6_Final.xml | 13 ++ .idea/libraries/Maven__jline_jline_0_9_94.xml | 13 ++ ...rg_apache_curator_curator_client_4_2_0.xml | 13 ++ ...apache_curator_curator_framework_4_2_0.xml | 13 ++ ...pache_yetus_audience_annotations_0_5_0.xml | 13 ++ ..._org_apache_zookeeper_zookeeper_3_4_14.xml | 13 ++ .idea/uiDesigner.xml | 124 ++++++++++++++++++ .../jlog/client/etcd/EtcdConfigFactory.java | 4 +- .../jlog/client/etcd/EtcdStarter.java | 16 +-- common/pom.xml | 29 ++++ .../jlog/common/config/IConfigCenter.java | 5 + .../jlog/common/config/etcd/JdEtcdClient.java | 10 ++ .../common/config/zookeeper/ZKClient.java | 122 +++++++++++++++++ .../common/config/zookeeper/ZkBuilder.java | 29 ++++ .../jlog/worker/config/EtcdConfig.java | 4 +- 17 files changed, 432 insertions(+), 11 deletions(-) create mode 100644 .idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml create mode 100644 .idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml create mode 100644 .idea/libraries/Maven__jline_jline_0_9_94.xml create mode 100644 .idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml create mode 100644 .idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml create mode 100644 .idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml create mode 100644 .idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml create mode 100644 .idea/uiDesigner.xml create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java diff --git a/ clientdemo/clientdemo.iml b/ clientdemo/clientdemo.iml index 086cbad..3b6a6e0 100644 --- a/ clientdemo/clientdemo.iml +++ b/ clientdemo/clientdemo.iml @@ -50,6 +50,14 @@ + + + + + + + + @@ -79,7 +87,6 @@ - diff --git a/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml b/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml new file mode 100644 index 0000000..8f8349e --- /dev/null +++ b/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml new file mode 100644 index 0000000..1bf9986 --- /dev/null +++ b/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__jline_jline_0_9_94.xml b/.idea/libraries/Maven__jline_jline_0_9_94.xml new file mode 100644 index 0000000..903c339 --- /dev/null +++ b/.idea/libraries/Maven__jline_jline_0_9_94.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml new file mode 100644 index 0000000..4c33315 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml new file mode 100644 index 0000000..0d8e1fe --- /dev/null +++ b/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml b/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml new file mode 100644 index 0000000..5237f9d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml new file mode 100644 index 0000000..124caa2 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..e96534f --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java index b2386b7..3e4b339 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java +++ b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java @@ -3,6 +3,7 @@ package com.jd.platform.jlog.client.etcd; import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; +import com.jd.platform.jlog.common.config.zookeeper.ZkBuilder; /** * @author wuweifeng wrote on 2020-01-07 @@ -19,6 +20,7 @@ public class EtcdConfigFactory { public static void buildConfigCenter(String etcdServer) { //连接多个时，逗号分隔 - configCenter = JdEtcdBuilder.build(etcdServer); + //configCenter = JdEtcdBuilder.build(etcdServer); + configCenter = ZkBuilder.build(etcdServer); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java index f7eb670..2aecf93 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java @@ -57,27 +57,27 @@ public class EtcdStarter { private void fetch() { IConfigCenter configCenter = EtcdConfigFactory.configCenter(); //获取所有worker的ip - List keyValues = null; + List keys = null; try { //如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有 if (Context.MDC != null) { String mdc = parseMdc(Context.MDC); - keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); + keys = configCenter.getPrefixKey(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); } - if (CollectionUtil.isEmpty(keyValues)) { - keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME); + if (CollectionUtil.isEmpty(keys)) { + keys = configCenter.getPrefixKey(Constant.WORKER_PATH + Context.APP_NAME); } //全是空，给个警告 - if (CollectionUtil.isEmpty(keyValues)) { + if (CollectionUtil.isEmpty(keys)) { logger.warn("very important warn !!! workers ip info is null!!!"); } List addresses = new ArrayList<>(); - if (keyValues != null) { - for (KeyValue keyValue : keyValues) { + if (keys != null) { + for (String key : keys) { //value里放的是ip地址 - String ipPort = keyValue.getValue().toStringUtf8(); + String ipPort = configCenter.get(key); addresses.add(ipPort); } } diff --git a/common/pom.xml b/common/pom.xml index b1551c1..11d6719 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -21,9 +21,38 @@ 5.1.0 0.0.16 1.7.2 + 3.4.14 + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + org.apache.curator + curator-framework + 4.2.0 + + + org.apache.zookeeper + zookeeper + + + io.protostuff protostuff-core diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java index d5e28b6..9177423 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java @@ -49,6 +49,11 @@ public interface IConfigCenter { */ List getPrefix(String key); + + /** + * 获取指定前缀的所有key + */ + List getPrefixKey(String key); /** * 监听key */ diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java index 9f2254c..08663ff 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java @@ -12,8 +12,10 @@ import com.ibm.etcd.client.lease.PersistentLease; import com.ibm.etcd.client.lock.LockClient; import com.jd.platform.jlog.common.config.IConfigCenter; +import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; @@ -121,6 +123,14 @@ public class JdEtcdClient implements IConfigCenter { return rangeResponse.getKvsList(); } + @Override + public List getPrefixKey(String key) { + RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); + return rangeResponse.getKvsList().stream().map( + kv -> kv.getKey().toStringUtf8() + ).collect(Collectors.toList()); + } + @Override public KvClient.WatchIterator watch(String key) { return kvClient.watch(ByteString.copyFromUtf8(key)).start(); diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java new file mode 100644 index 0000000..f4327d5 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java @@ -0,0 +1,122 @@ +package com.jd.platform.jlog.common.config.zookeeper; + +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.client.kv.KvClient; +import com.jd.platform.jlog.common.config.IConfigCenter; +import org.apache.curator.framework.CuratorFramework; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; + +import java.util.List; + +/** + * @author shenkaiwen5 + * @version 1.0 + * @date 2022-01-06 + */ +public class ZKClient implements IConfigCenter { + + private CuratorFramework curator; + + public ZKClient(CuratorFramework curatorFramework) { + this.curator = curatorFramework; + } + + @Override + public void put(String key, String value) { + try { + curator.create().creatingParentsIfNeeded() // 若创建节点的父节点不存在则先创建父节点再创建子节点 + .withMode(CreateMode.EPHEMERAL) // 创建的是临时节点 + .withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE) // 默认匿名权限,权限scheme id:'world,'anyone,:cdrwa + .forPath(key, value.getBytes()); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void put(String key, String value, long leaseId) { + + } + + @Override + public void revoke(long leaseId) { + + } + + @Override + public long putAndGrant(String key, String value, long ttl) { + return 0; + } + + @Override + public long setLease(String key, long leaseId) { + return 0; + } + + @Override + public void delete(String key) { + + } + + @Override + public String get(String key) { + try { + return curator.getData().forPath(key).toString(); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Override + public List getPrefix(String key) { + return null; + } + + @Override + public List getPrefixKey(String key) { + try { + return curator.getChildren().forPath(key); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Override + public KvClient.WatchIterator watch(String key) { + return null; + } + + @Override + public KvClient.WatchIterator watchPrefix(String key) { + return null; + } + + @Override + public long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception { + return 0; + } + + @Override + public long buildAliveLease(int frequencySecs, int minTtl) throws Exception { + return 0; + } + + @Override + public long buildNormalLease(long ttl) { + return 0; + } + + @Override + public long timeToLive(long leaseId) { + return 0; + } + + @Override + public KeyValue getKv(String key) { + return null; + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java new file mode 100644 index 0000000..a5772fb --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java @@ -0,0 +1,29 @@ +package com.jd.platform.jlog.common.config.zookeeper; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; + +/** + * @author shenkaiwen5 + * @version 1.0 + * @date 2022-01-06 + */ +public class ZkBuilder { + /** + * 构建ZKClient + */ + public static ZKClient build(String endPoints) { + CuratorFramework client = CuratorFrameworkFactory.builder().connectString(endPoints) + // 连接超时时间 + .sessionTimeoutMs(1000) + // 会话超时时间 + .connectionTimeoutMs(1000) + // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + client.start(); + + return new ZKClient(client); + } +} diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java index 4195e9a..3c8a7f1 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.worker.config; import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; +import com.jd.platform.jlog.common.config.zookeeper.ZkBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -25,7 +26,8 @@ public class EtcdConfig { public IConfigCenter client() { logger.info("etcd address : " + etcdServer); //连接多个时，逗号分隔 - return JdEtcdBuilder.build(etcdServer); + //return JdEtcdBuilder.build(etcdServer); + return ZkBuilder.build(etcdServer); } } -- Gitee From 04794d0bf8bfd041d9837e62c9344743dbf9e8da Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Thu, 6 Jan 2022 21:47:56 +0800 Subject: [PATCH 02/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/jlog/worker/config/EtcdStarter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java index 67aac4a..5a514a0 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java @@ -50,8 +50,10 @@ public class EtcdStarter { scheduledExecutorService.scheduleAtFixedRate(() -> { try { - configCenter.putAndGrant(buildKey(), buildValue(), 8); - configCenter.putAndGrant(buildSecondKey(), buildValue(), 8); + //configCenter.putAndGrant(buildKey(), buildValue(), 8); + //configCenter.putAndGrant(buildSecondKey(), buildValue(), 8); + configCenter.put(buildKey(), buildValue()); + configCenter.put(buildSecondKey(), buildValue()); } catch (Exception e) { //do nothing e.printStackTrace(); -- Gitee From 09912ca957fbb063e9f257453fe032c4eda247da Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Fri, 7 Jan 2022 10:54:19 +0800 Subject: [PATCH 03/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jd/platform/jlog/common/config/IConfigCenter.java | 6 ------ .../platform/jlog/common/config/etcd/JdEtcdClient.java | 10 +++++----- .../jlog/common/config/zookeeper/ZKClient.java | 4 ---- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java index 9177423..8ebd4d3 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java @@ -44,12 +44,6 @@ public interface IConfigCenter { */ String get(String key); - /** - * 获取指定前缀的所有key-value - */ - List getPrefix(String key); - - /** * 获取指定前缀的所有key */ diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java index 08663ff..e274685 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java @@ -117,11 +117,11 @@ public class JdEtcdClient implements IConfigCenter { return keyValues.get(0); } - @Override - public List getPrefix(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); - return rangeResponse.getKvsList(); - } +// @Override +// public List getPrefix(String key) { +// RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); +// return rangeResponse.getKvsList(); +// } @Override public List getPrefixKey(String key) { diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java index f4327d5..c257bfe 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java @@ -70,10 +70,6 @@ public class ZKClient implements IConfigCenter { return null; } - @Override - public List getPrefix(String key) { - return null; - } @Override public List getPrefixKey(String key) { -- Gitee From 48766c21badfb782b54c20f7bbde6ce72773893b Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Fri, 7 Jan 2022 11:22:59 +0800 Subject: [PATCH 04/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- { clientdemo => clientdemo}/clientdemo.iml | 0 { clientdemo => clientdemo}/pom.xml | 0 .../clientdemo/ClientDemoApplication.java | 0 .../jlog/clientdemo/config/DemoConfig.java | 0 .../jlog/clientdemo/web/TestController.java | 0 .../src/main/resources/application.yml | 0 .../src/main/resources/log4j2.xml | 0 .../platform/jlog/test/TracerPacketTest.java | 0 clientdemo/target/classes/application.yml | 5 ++ .../clientdemo/ClientDemoApplication.class | Bin 0 -> 781 bytes .../jlog/clientdemo/config/DemoConfig.class | Bin 0 -> 1995 bytes .../jlog/clientdemo/web/TestController.class | Bin 0 -> 1113 bytes clientdemo/target/classes/log4j2.xml | 54 ++++++++++++++++++ .../jlog/test/TracerPacketTest$MyRes.class | Bin 0 -> 691 bytes .../platform/jlog/test/TracerPacketTest.class | Bin 0 -> 3459 bytes 15 files changed, 59 insertions(+) rename { clientdemo => clientdemo}/clientdemo.iml (100%) rename { clientdemo => clientdemo}/pom.xml (100%) rename { clientdemo => clientdemo}/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java (100%) rename { clientdemo => clientdemo}/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java (100%) rename { clientdemo => clientdemo}/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java (100%) rename { clientdemo => clientdemo}/src/main/resources/application.yml (100%) rename { clientdemo => clientdemo}/src/main/resources/log4j2.xml (100%) rename { clientdemo => clientdemo}/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java (100%) create mode 100644 clientdemo/target/classes/application.yml create mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class create mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/config/DemoConfig.class create mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/web/TestController.class create mode 100644 clientdemo/target/classes/log4j2.xml create mode 100644 clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class create mode 100644 clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class diff --git a/ clientdemo/clientdemo.iml b/clientdemo/clientdemo.iml similarity index 100% rename from clientdemo/clientdemo.iml rename to clientdemo/clientdemo.iml diff --git a/ clientdemo/pom.xml b/clientdemo/pom.xml similarity index 100% rename from clientdemo/pom.xml rename to clientdemo/pom.xml diff --git a/ clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java similarity index 100% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java rename to clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java diff --git a/ clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java similarity index 100% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java rename to clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java diff --git a/ clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java similarity index 100% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java rename to clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java diff --git a/ clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml similarity index 100% rename from clientdemo/src/main/resources/application.yml rename to clientdemo/src/main/resources/application.yml diff --git a/ clientdemo/src/main/resources/log4j2.xml b/clientdemo/src/main/resources/log4j2.xml similarity index 100% rename from clientdemo/src/main/resources/log4j2.xml rename to clientdemo/src/main/resources/log4j2.xml diff --git a/ clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java b/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java similarity index 100% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java rename to clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java diff --git a/clientdemo/target/classes/application.yml b/clientdemo/target/classes/application.yml new file mode 100644 index 0000000..bf94e96 --- /dev/null +++ b/clientdemo/target/classes/application.yml @@ -0,0 +1,5 @@ +#etcd的地址，如有多个用逗号分隔 +config: + server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ +server: + port: 8081 \ No newline at end of file diff --git a/clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class b/clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class new file mode 100644 index 0000000000000000000000000000000000000000..55c6b14f1e56ed1c7332bf38fd35d59bfa0aff74 GIT binary patch literal 781 zcmbVK%Tn7g6g>(F32w?GP+lzyb|DM2Y3U*iGc=v93}r|sU34Woiczqoj4adsR#z}A z_<(*?r&op6p(_;U>nCotlKlsgl}_ zWoG!GG(VGhI-8{`7ET%6r*WB3nqjm1Bbf|rS2Tz3k%~wWH*r)s0M~&W<|7u{KJjo_` zoHQfd=-q<-F4HZNJ`F5miDdh+00CZ+w?zJHYT^$cmCb-wr`Ef}!uRL%0cC(Ss_ImF aw5a0>#aHQXFmgPawNiGlPTG&%0R91W=%7!sV0UtHjV8#lYm?j?=m z_|gBPGY*|Gqd&kO<#_HcrLjS#jvv_j+{<&%Is3frpMU-K2Y?lPWuk!PG*(O`u$;ji zy?&U*6h6|{$NEt*aSM0TxM#xDzV}U7Sk{{dX?&8wsy5d$Sl8=A{dknd<1{`sU`o|& z)nsra0|WW$fqi5P*Y-N1rUJ+7R193%?Rm;MlyxU`8m?URJYU($@x9Q%O4ScKBJ66{ zc3>aMr+)CAXh_=&McZyFKlonQ!waHryFE$Bi6dv)d3 zZy;GL)eWT9{T3ZBRvk}n^$r^{*t01@OZA(!Teky8?+1q|^{qoyx2jG5P#m;G*R@sK z4`{o7M>JhWda5N4eG#*YhqTut0a{c;b{x;ZB^B6B8LUU%HDw1%hh}LUrr0}sg@|)e zIFC<9V8;&?8ABEHnu;#jUf@1<+=${qu{1iIzOTen*$_HNu2dqvuuHkbiBhq~h!f2U zPMsOJ_dnsx9vPpj6JrC=j^07+4=+&7&5`sBy^_c3bWjHB&6?i}nv#S%<1ccvsEg9T z?N^11Gun$IQ?c+SX4Cl0z@0HwF)D-eYdyzp>Ex$}mbKw|x;E@El&rLrt|ki`_}s!Y zUbpatwyM~)u!UIzm67f7YWZXU#|b_YaSPknVNbo>!Uhsz!NQl=WmI*7qGFBHYN3Wb z3;STJEsbFt-8BCo+io1lrZT|Z`q$nY6=jZ~P^#5#cZ=i0z-IC63@VK+y%D|lJaa6< z^8|l%YQ#-6bW%AZ*lhn%X@sWd!H(0FE(a3Zm*{aMIVWS~$1k;MMUSxycB{1?xI4CD zr|ZF4D1JTSJ?WVyVjPL7QIViWh928NRF|pZMyZas`7J2$Tfs!@BIJ67yK`LM;BO|x zRU~P!4fuSEe+jMz*NKH^Fn*ztz#N~G(cxv{fuE1a0oQPyI|K8)oQN)e!!^OPt0zc2 zd4^;@)hE!enCv6nN2XftBl`lgfC>7}AWIQBN|+|j3~@9^A;uYSOw;onyh|>vUTb-e ztAQd)1LmLTk>r^;fl20b)e~4;&u^A*^l_n_`W+XaB=fmiAFq}3`93a{DeZFkXI_b4 za29FW*OB4t24D-5dNIz*lf24>D>aQ1xk8Q(-I`4R!C5(5>f!EtttV(oQ<<=*tNsPY2gj{ z&x>$KltbbHcn@9!F;1e2hJb{%>lu&7GoQa1fB*UA8-NmSx$v-0Ek#eZy!&Y38$$_XuWTC^phz>;(+I82MHqXK)L#gcRnAfYk5eZY%3F(mz zxF3a5nW~I6Kb94KKsa`^GKr3&cIr&1s`4encyYO07e|6eLIwO$r7nF#$Xt^MUncjY zPz>vTNulFdD;^W*h##E#)9IPLmNH>155s0ifbS}$jWD5Bg!gubh(;2spq7YO9_!?J z8)JnaadGO)yYxK}jRyHr*vyfLT0}>dcfmW`bj=Qf?!Im%zT6EhUt?#lyKXP|f0@s) z}3+&^0# z3-nMtdPvk+mJ8AbNGDopK1VA~l99};oPfO}C4~ab*|u>6W2Ac>2aIEaRt%GvA}@RR z3CR@6*{?`FKEXhFwUGXZ%zI=DxzBJ?Sba;oorndbttTAJ5sV9%#(X>aWG6y5xnPbY6r8{1>hG5 CO%{a! literal 0 HcmV?d00001 diff --git a/clientdemo/target/classes/log4j2.xml b/clientdemo/target/classes/log4j2.xml new file mode 100644 index 0000000..00f41b9 --- /dev/null +++ b/clientdemo/target/classes/log4j2.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class b/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class new file mode 100644 index 0000000000000000000000000000000000000000..eaf0b2e05dc5c28d23450dd65cc52d71e17b8eb6 GIT binary patch literal 691 zcmb7CO-sW-5Ph4r#>Qx3{r>jgL2EHZyeWbRf)H)BwfAkh*3>kS?N;>WU-D1{5B>mu zlsFp&K`-?%J2P*2Z(nBe{_*+-U=M3~WHDzUhk0Esm{>Bg%wPsdti*$2DEJ+I&%KDp zZLg+e7`OKsM$}E%+bA=X{2=LiotAeOadn+Ys-vXsDbZ72UGhN4Di3ajsuQ4j2Voql zLx!Wx{}BIL*lI9jkCT>QaQrY9=lyO|$U1LEgcSTF;8BCiQ1{O3>FZBD!Zn-6X9lm%J;KkOWz^q%Q+;5^66qzo)RH$7y2}c^ej1 zY>dD(vC2>$ev)4f=<%P`RGJ+T(7H04dNjr1wXjZm&(Y6dP>>hEb+tR{7=?u~y0NLc zqMV`Z6t@Rp#TOXG(g2xff((q)Igwshh+3ukf&wI-QsAIK9g$OME$m>59y2gab>6_} VlS_b>nn}-DOj5SULl%p0flsv=sxSZm literal 0 HcmV?d00001 diff --git a/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class b/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class new file mode 100644 index 0000000000000000000000000000000000000000..75a1963e59f2e8ac92bf32f8dc5bdb3a57b07153 GIT binary patch literal 3459 zcma)9S#;c18U8f(ScA(N|_f_+k z|Gab)z!CgO#}F>7cvMG5ye%D#xGW9@6}Ao)MGcOI#bzvFS%a%%1y?kbG?X=XDtsM# z1j8Ow#N#pX2sDHmA`MjyS2aAY;hKh34e!?Qo@Trk@6+*qJRy`kDIOotF^CWD!iVr- z9Us9*b$kp@>G(K4q2XyApTwtBd|F30KBMBZIzETbYxshWUVKr9g6#C_ zIVV(LOnROZOq8rJ6vw6$uIEOh3K}}Q78Ep2_(exS%arRmGu3jz3FfRqiJ07!Z(F4W zD{#d*K{iIqE@kvi*?!qvDVmj%6)pM!;gY{(Mot)+bAe?$!NZn)#fj#K7*WtHE@mCC zIA5&L9^3aK$BV{SiB^!gG)`L!@u9h3h0f6<$yIaUgcaWlooS1FEmJGjRm-eKZpkEq zauu4{7Gq16w`4wW&2}o0>wCP@BWcIDV|fbtx3`y0lzq=EGXkZ=Zb46OBj!F^SaIw~ zL2I4rDOdWW2hL+vx+P7!V{5EdD`~v$t79bPi8^Cvqrml+M(QxJwvkclf3S{EC`=er z-(RO3A!+w)#8gnX%%H4|M^$`P#n)Iw3i9WwUgVaY1vhk=<}uInBTIyknQ5K!gC%pN z%5pK2OzJX|(sBj2cXriTpZLSb&-&HCc22pXirQ{1hklVn1xKaqup(lx7+7WJaX+|X z7JNU7t4P+!waA>66ywAv7N~Yw$j5d_YRP4B6`itgPDt=1!LdrEKV?55Wo1WXmuLSza;l1@G<{wcrFH zONAtZ1O12lhuF+68+F&<(6Rmj{sw7e=mb}tV3O#DYT#14h#(8lz;pPKiXR*J37$9b zQ}Os2o;5I!1r({}~+KhcVVs6_^vq(N;yor!ArX}P&l`}16+1q)`lo*i%y>n8$8Y)!^Ii8Tnuq5-P zu4fZWeH$kz1$73^&rhCYUkA*;UycvCwD;vm5;(R_6N`<}QqWhgHW>`{O12an8Mn}H zW7MtmTI9zl`e_xZu&rKUD&XRlK+mp*kyBUMX+T4S-S#UioST)DxBIYJ^Jb_P-UE6Dy6T)PBm|XE<(>np@fWNrt1n~wQ z?A&2m@34YJ3Ac#ZAIbeC3tzzw+G829?~|3p^DJX3#?IIdnzwEL`gI&9NA^@+cx{#3 zf>p50MJtR}LVh=#o;^DwJ{5ccq^Uf#T_x7FTp|@6{KgyNCnwMVMeGbdkMU&kt?-@g zUW0O-KnBP8Y?8=3@fN~HOn_mG@TA}bnKObpa{yUr#Ql2%nG0)ZnA$)i-`VM&-Wr-V zpk8QgU4u5$m)n)oZ=ksW=lgQTI(Dbf^5$}_HS8rx4Y$|Omfqfx+n0iG z$mLUTf9}o{eB&ngrrcdIwzuyF4rFk?XC3W5HFP*n)$nHedH0JrSi?QTjrqoP92(B% zv+KBbIJQ@#y~Ao=&2{FR5-EB0H8EY!K^JT_^jv2op2sWrCqIlEWz1HvpRqW=h_s`X z0dK?IJRRiyJ&f2PjNx8fMkhbuyHF+eN%Y`penvloKCaRMN;`}-Qr={2|4fN5Gx~pH z%>TiNzk>Uz>8*GbqxcuzCiC(Lc9DLF*il+=lhW=dlqK&$Qj9W(FC$N_i|A-VQ~@xhBl=T9ncp-x;xL35%*bLvuvcb!?#bLU)q+b@Vayxqd#F`8Am9 z75ywg Date: Fri, 7 Jan 2022 11:47:21 +0800 Subject: [PATCH 05/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clientdemo/clientdemo.iml | 146 -------------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 clientdemo/clientdemo.iml diff --git a/clientdemo/clientdemo.iml b/clientdemo/clientdemo.iml deleted file mode 100644 index 3b6a6e0..0000000 --- a/clientdemo/clientdemo.iml +++ /dev/null @@ -1,146 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file -- Gitee From 4cbac0eedace178a7fe7bffa34a2efbe65ee4ed6 Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Fri, 7 Jan 2022 11:50:24 +0800 Subject: [PATCH 06/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ -- Gitee From 34fc88a74ee16da0ad45b81da7d9c2eb9754a338 Mon Sep 17 00:00:00 2001 From: shenkaiwen5 Date: Fri, 7 Jan 2022 14:22:52 +0800 Subject: [PATCH 07/22] =?UTF-8?q?master=20=20=E4=B8=8D=E4=BD=BF=E7=94=A8et?= =?UTF-8?q?cd=E4=BD=BF=E7=94=A8zookeeper=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ub_spotbugs_spotbugs_annotations_3_1_9.xml | 13 -- ...en__com_squareup_okhttp3_okhttp_3_14_9.xml | 13 -- .../Maven__com_squareup_okio_okio_1_17_2.xml | 13 -- .../Maven__io_netty_netty_3_10_6_Final.xml | 13 -- .idea/libraries/Maven__jline_jline_0_9_94.xml | 13 -- .idea/libraries/Maven__junit_junit_4_12.xml | 13 -- ...rg_apache_curator_curator_client_4_2_0.xml | 13 -- ...apache_curator_curator_framework_4_2_0.xml | 13 -- ...apache_logging_log4j_log4j_core_2_14_1.xml | 13 -- ..._logging_log4j_log4j_slf4j_impl_2_14_1.xml | 13 -- ...pache_yetus_audience_annotations_0_5_0.xml | 13 -- ..._org_apache_zookeeper_zookeeper_3_4_14.xml | 13 -- .../Maven__org_hamcrest_hamcrest_core_2_2.xml | 13 -- .idea/uiDesigner.xml | 124 ------------------ client/pom.xml | 5 + clientdemo/target/classes/application.yml | 5 - .../clientdemo/ClientDemoApplication.class | Bin 781 -> 0 bytes .../jlog/clientdemo/config/DemoConfig.class | Bin 1995 -> 0 bytes .../jlog/clientdemo/web/TestController.class | Bin 1113 -> 0 bytes clientdemo/target/classes/log4j2.xml | 54 -------- .../jlog/test/TracerPacketTest$MyRes.class | Bin 691 -> 0 bytes .../platform/jlog/test/TracerPacketTest.class | Bin 3459 -> 0 bytes 22 files changed, 5 insertions(+), 352 deletions(-) delete mode 100644 .idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml delete mode 100644 .idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_14_9.xml delete mode 100644 .idea/libraries/Maven__com_squareup_okio_okio_1_17_2.xml delete mode 100644 .idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml delete mode 100644 .idea/libraries/Maven__jline_jline_0_9_94.xml delete mode 100644 .idea/libraries/Maven__junit_junit_4_12.xml delete mode 100644 .idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_14_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_14_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml delete mode 100644 .idea/libraries/Maven__org_hamcrest_hamcrest_core_2_2.xml delete mode 100644 .idea/uiDesigner.xml delete mode 100644 clientdemo/target/classes/application.yml delete mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class delete mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/config/DemoConfig.class delete mode 100644 clientdemo/target/classes/com/jd/platform/jlog/clientdemo/web/TestController.class delete mode 100644 clientdemo/target/classes/log4j2.xml delete mode 100644 clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class delete mode 100644 clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class diff --git a/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml b/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml deleted file mode 100644 index 8f8349e..0000000 --- a/.idea/libraries/Maven__com_github_spotbugs_spotbugs_annotations_3_1_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_14_9.xml b/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_14_9.xml deleted file mode 100644 index c7878f4..0000000 --- a/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_14_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okio_okio_1_17_2.xml b/.idea/libraries/Maven__com_squareup_okio_okio_1_17_2.xml deleted file mode 100644 index 9260035..0000000 --- a/.idea/libraries/Maven__com_squareup_okio_okio_1_17_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml deleted file mode 100644 index 1bf9986..0000000 --- a/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__jline_jline_0_9_94.xml b/.idea/libraries/Maven__jline_jline_0_9_94.xml deleted file mode 100644 index 903c339..0000000 --- a/.idea/libraries/Maven__jline_jline_0_9_94.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml deleted file mode 100644 index d411041..0000000 --- a/.idea/libraries/Maven__junit_junit_4_12.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml deleted file mode 100644 index 4c33315..0000000 --- a/.idea/libraries/Maven__org_apache_curator_curator_client_4_2_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml deleted file mode 100644 index 0d8e1fe..0000000 --- a/.idea/libraries/Maven__org_apache_curator_curator_framework_4_2_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_14_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_14_1.xml deleted file mode 100644 index 1b2d975..0000000 --- a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_14_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_14_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_14_1.xml deleted file mode 100644 index 22506f0..0000000 --- a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_14_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml b/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml deleted file mode 100644 index 5237f9d..0000000 --- a/.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml deleted file mode 100644 index 124caa2..0000000 --- a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_14.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_2_2.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_2_2.xml deleted file mode 100644 index 15f1e4c..0000000 --- a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_2_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml deleted file mode 100644 index e96534f..0000000 --- a/.idea/uiDesigner.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/client/pom.xml b/client/pom.xml index b6e665c..6e43109 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -24,6 +24,11 @@ 2.4 provided + + org.slf4j + slf4j-log4j12 + 1.7.32 + com.alibaba diff --git a/clientdemo/target/classes/application.yml b/clientdemo/target/classes/application.yml deleted file mode 100644 index bf94e96..0000000 --- a/clientdemo/target/classes/application.yml +++ /dev/null @@ -1,5 +0,0 @@ -#etcd的地址，如有多个用逗号分隔 -config: - server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ -server: - port: 8081 \ No newline at end of file diff --git a/clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class b/clientdemo/target/classes/com/jd/platform/jlog/clientdemo/ClientDemoApplication.class deleted file mode 100644 index 55c6b14f1e56ed1c7332bf38fd35d59bfa0aff74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 781 zcmbVK%Tn7g6g>(F32w?GP+lzyb|DM2Y3U*iGc=v93}r|sU34Woiczqoj4adsR#z}A z_<(*?r&op6p(_;U>nCotlKlsgl}_ zWoG!GG(VGhI-8{`7ET%6r*WB3nqjm1Bbf|rS2Tz3k%~wWH*r)s0M~&W<|7u{KJjo_` zoHQfd=-q<-F4HZNJ`F5miDdh+00CZ+w?zJHYT^$cmCb-wr`Ef}!uRL%0cC(Ss_ImF aw5a0>#aHQXFmgPawNiGlPTG&%0R91W=%7!sV0UtHjV8#lYm?j?=m z_|gBPGY*|Gqd&kO<#_HcrLjS#jvv_j+{<&%Is3frpMU-K2Y?lPWuk!PG*(O`u$;ji zy?&U*6h6|{$NEt*aSM0TxM#xDzV}U7Sk{{dX?&8wsy5d$Sl8=A{dknd<1{`sU`o|& z)nsra0|WW$fqi5P*Y-N1rUJ+7R193%?Rm;MlyxU`8m?URJYU($@x9Q%O4ScKBJ66{ zc3>aMr+)CAXh_=&McZyFKlonQ!waHryFE$Bi6dv)d3 zZy;GL)eWT9{T3ZBRvk}n^$r^{*t01@OZA(!Teky8?+1q|^{qoyx2jG5P#m;G*R@sK z4`{o7M>JhWda5N4eG#*YhqTut0a{c;b{x;ZB^B6B8LUU%HDw1%hh}LUrr0}sg@|)e zIFC<9V8;&?8ABEHnu;#jUf@1<+=${qu{1iIzOTen*$_HNu2dqvuuHkbiBhq~h!f2U zPMsOJ_dnsx9vPpj6JrC=j^07+4=+&7&5`sBy^_c3bWjHB&6?i}nv#S%<1ccvsEg9T z?N^11Gun$IQ?c+SX4Cl0z@0HwF)D-eYdyzp>Ex$}mbKw|x;E@El&rLrt|ki`_}s!Y zUbpatwyM~)u!UIzm67f7YWZXU#|b_YaSPknVNbo>!Uhsz!NQl=WmI*7qGFBHYN3Wb z3;STJEsbFt-8BCo+io1lrZT|Z`q$nY6=jZ~P^#5#cZ=i0z-IC63@VK+y%D|lJaa6< z^8|l%YQ#-6bW%AZ*lhn%X@sWd!H(0FE(a3Zm*{aMIVWS~$1k;MMUSxycB{1?xI4CD zr|ZF4D1JTSJ?WVyVjPL7QIViWh928NRF|pZMyZas`7J2$Tfs!@BIJ67yK`LM;BO|x zRU~P!4fuSEe+jMz*NKH^Fn*ztz#N~G(cxv{fuE1a0oQPyI|K8)oQN)e!!^OPt0zc2 zd4^;@)hE!enCv6nN2XftBl`lgfC>7}AWIQBN|+|j3~@9^A;uYSOw;onyh|>vUTb-e ztAQd)1LmLTk>r^;fl20b)e~4;&u^A*^l_n_`W+XaB=fmiAFq}3`93a{DeZFkXI_b4 za29FW*OB4t24D-5dNIz*lf24>D>aQ1xk8Q(-I`4R!C5(5>f!EtttV(oQ<<=*tNsPY2gj{ z&x>$KltbbHcn@9!F;1e2hJb{%>lu&7GoQa1fB*UA8-NmSx$v-0Ek#eZy!&Y38$$_XuWTC^phz>;(+I82MHqXK)L#gcRnAfYk5eZY%3F(mz zxF3a5nW~I6Kb94KKsa`^GKr3&cIr&1s`4encyYO07e|6eLIwO$r7nF#$Xt^MUncjY zPz>vTNulFdD;^W*h##E#)9IPLmNH>155s0ifbS}$jWD5Bg!gubh(;2spq7YO9_!?J z8)JnaadGO)yYxK}jRyHr*vyfLT0}>dcfmW`bj=Qf?!Im%zT6EhUt?#lyKXP|f0@s) z}3+&^0# z3-nMtdPvk+mJ8AbNGDopK1VA~l99};oPfO}C4~ab*|u>6W2Ac>2aIEaRt%GvA}@RR z3CR@6*{?`FKEXhFwUGXZ%zI=DxzBJ?Sba;oorndbttTAJ5sV9%#(X>aWG6y5xnPbY6r8{1>hG5 CO%{a! diff --git a/clientdemo/target/classes/log4j2.xml b/clientdemo/target/classes/log4j2.xml deleted file mode 100644 index 00f41b9..0000000 --- a/clientdemo/target/classes/log4j2.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class b/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest$MyRes.class deleted file mode 100644 index eaf0b2e05dc5c28d23450dd65cc52d71e17b8eb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 691 zcmb7CO-sW-5Ph4r#>Qx3{r>jgL2EHZyeWbRf)H)BwfAkh*3>kS?N;>WU-D1{5B>mu zlsFp&K`-?%J2P*2Z(nBe{_*+-U=M3~WHDzUhk0Esm{>Bg%wPsdti*$2DEJ+I&%KDp zZLg+e7`OKsM$}E%+bA=X{2=LiotAeOadn+Ys-vXsDbZ72UGhN4Di3ajsuQ4j2Voql zLx!Wx{}BIL*lI9jkCT>QaQrY9=lyO|$U1LEgcSTF;8BCiQ1{O3>FZBD!Zn-6X9lm%J;KkOWz^q%Q+;5^66qzo)RH$7y2}c^ej1 zY>dD(vC2>$ev)4f=<%P`RGJ+T(7H04dNjr1wXjZm&(Y6dP>>hEb+tR{7=?u~y0NLc zqMV`Z6t@Rp#TOXG(g2xff((q)Igwshh+3ukf&wI-QsAIK9g$OME$m>59y2gab>6_} VlS_b>nn}-DOj5SULl%p0flsv=sxSZm diff --git a/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class b/clientdemo/target/test-classes/com/jd/platform/jlog/test/TracerPacketTest.class deleted file mode 100644 index 75a1963e59f2e8ac92bf32f8dc5bdb3a57b07153..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3459 zcma)9S#;c18U8f(ScA(N|_f_+k z|Gab)z!CgO#}F>7cvMG5ye%D#xGW9@6}Ao)MGcOI#bzvFS%a%%1y?kbG?X=XDtsM# z1j8Ow#N#pX2sDHmA`MjyS2aAY;hKh34e!?Qo@Trk@6+*qJRy`kDIOotF^CWD!iVr- z9Us9*b$kp@>G(K4q2XyApTwtBd|F30KBMBZIzETbYxshWUVKr9g6#C_ zIVV(LOnROZOq8rJ6vw6$uIEOh3K}}Q78Ep2_(exS%arRmGu3jz3FfRqiJ07!Z(F4W zD{#d*K{iIqE@kvi*?!qvDVmj%6)pM!;gY{(Mot)+bAe?$!NZn)#fj#K7*WtHE@mCC zIA5&L9^3aK$BV{SiB^!gG)`L!@u9h3h0f6<$yIaUgcaWlooS1FEmJGjRm-eKZpkEq zauu4{7Gq16w`4wW&2}o0>wCP@BWcIDV|fbtx3`y0lzq=EGXkZ=Zb46OBj!F^SaIw~ zL2I4rDOdWW2hL+vx+P7!V{5EdD`~v$t79bPi8^Cvqrml+M(QxJwvkclf3S{EC`=er z-(RO3A!+w)#8gnX%%H4|M^$`P#n)Iw3i9WwUgVaY1vhk=<}uInBTIyknQ5K!gC%pN z%5pK2OzJX|(sBj2cXriTpZLSb&-&HCc22pXirQ{1hklVn1xKaqup(lx7+7WJaX+|X z7JNU7t4P+!waA>66ywAv7N~Yw$j5d_YRP4B6`itgPDt=1!LdrEKV?55Wo1WXmuLSza;l1@G<{wcrFH zONAtZ1O12lhuF+68+F&<(6Rmj{sw7e=mb}tV3O#DYT#14h#(8lz;pPKiXR*J37$9b zQ}Os2o;5I!1r({}~+KhcVVs6_^vq(N;yor!ArX}P&l`}16+1q)`lo*i%y>n8$8Y)!^Ii8Tnuq5-P zu4fZWeH$kz1$73^&rhCYUkA*;UycvCwD;vm5;(R_6N`<}QqWhgHW>`{O12an8Mn}H zW7MtmTI9zl`e_xZu&rKUD&XRlK+mp*kyBUMX+T4S-S#UioST)DxBIYJ^Jb_P-UE6Dy6T)PBm|XE<(>np@fWNrt1n~wQ z?A&2m@34YJ3Ac#ZAIbeC3tzzw+G829?~|3p^DJX3#?IIdnzwEL`gI&9NA^@+cx{#3 zf>p50MJtR}LVh=#o;^DwJ{5ccq^Uf#T_x7FTp|@6{KgyNCnwMVMeGbdkMU&kt?-@g zUW0O-KnBP8Y?8=3@fN~HOn_mG@TA}bnKObpa{yUr#Ql2%nG0)ZnA$)i-`VM&-Wr-V zpk8QgU4u5$m)n)oZ=ksW=lgQTI(Dbf^5$}_HS8rx4Y$|Omfqfx+n0iG z$mLUTf9}o{eB&ngrrcdIwzuyF4rFk?XC3W5HFP*n)$nHedH0JrSi?QTjrqoP92(B% zv+KBbIJQ@#y~Ao=&2{FR5-EB0H8EY!K^JT_^jv2op2sWrCqIlEWz1HvpRqW=h_s`X z0dK?IJRRiyJ&f2PjNx8fMkhbuyHF+eN%Y`penvloKCaRMN;`}-Qr={2|4fN5Gx~pH z%>TiNzk>Uz>8*GbqxcuzCiC(Lc9DLF*il+=lhW=dlqK&$Qj9W(FC$N_i|A-VQ~@xhBl=T9ncp-x;xL35%*bLvuvcb!?#bLU)q+b@Vayxqd#F`8Am9 z75ywg Date: Wed, 16 Feb 2022 11:35:57 +0800 Subject: [PATCH 08/22] =?UTF-8?q?=E5=A4=9A=E9=85=8D=E7=BD=AE=E4=B8=AD?= =?UTF-8?q?=E5=BF=83=E6=94=AF=E6=8C=81=EF=BC=8C=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?tag=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dashboard/pom.xml | 5 - .../src/main/resources/logback-spring.xml | 2 +- .../jlog/client/TracerClientStarter.java | 43 +++-- .../jlog/client/etcd/EtcdConfigFactory.java | 26 --- .../{UserFilter.java => HttpFilter.java} | 8 +- .../EtcdStarter.java => task/Monitor.java} | 29 +-- clientdemo/pom.xml | 15 +- .../clientdemo/ClientDemoApplication.java | 8 +- .../jlog/clientdemo/config/DemoConfig.java | 47 ++++- .../config/TracerLogbackAppender.java | 119 ++++++++++++ .../jlog/clientdemo/web/TestController.java | 21 ++- clientdemo/src/main/resources/application.yml | 21 ++- clientdemo/src/main/resources/log4j2.xml | 54 ------ .../src/main/resources/logback-spring.xml | 62 ++++++ .../platform/jlog/test/TracerPacketTest.java | 10 +- common/pom.xml | 16 ++ .../jlog/common/config/ConfigCenterEnum.java | 15 ++ .../common/config/ConfigCenterFactory.java | 58 ++++++ .../jlog/common/config/IConfigCenter.java | 79 ++------ .../common/config/apollo/ApolloClient.java | 57 ++++++ .../common/config/etcd/JdEtcdBuilder.java | 18 -- .../jlog/common/config/etcd/JdEtcdClient.java | 128 ++----------- .../jlog/common/config/nacos/NacosClient.java | 68 +++++++ .../common/config/zookeeper/ZKClient.java | 82 +++----- .../common/config/zookeeper/ZkBuilder.java | 29 --- .../config/zookeeper/ZookeeperTest.java | 19 ++ .../jlog/common/constant/Constant.java | 31 +++ .../jlog/common/model/CenterConfig.java | 55 ++++++ .../platform/jlog/common/model/TagConfig.java | 126 +++++++++++++ .../jlog/common/model/TagHandler.java | 178 ++++++++++++++++++ .../jlog/common/utils/ConfigUtil.java | 67 +++++++ worker/pom.xml | 6 + .../com/jd/platform/jlog/worker/Starter.java | 6 +- .../{EtcdStarter.java => CenterStarter.java} | 2 +- .../{EtcdConfig.java => ConfigCenter.java} | 42 ++++- .../platform/jlog/worker/config/IConfig.java | 10 - worker/src/main/resources/application.yml | 2 +- 37 files changed, 1119 insertions(+), 445 deletions(-) delete mode 100644 client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java rename client/src/main/java/com/jd/platform/jlog/client/filter/{UserFilter.java => HttpFilter.java} (97%) rename client/src/main/java/com/jd/platform/jlog/client/{etcd/EtcdStarter.java => task/Monitor.java} (81%) create mode 100644 clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java delete mode 100644 clientdemo/src/main/resources/log4j2.xml create mode 100644 clientdemo/src/main/resources/logback-spring.xml create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java rename worker/src/main/java/com/jd/platform/jlog/worker/config/{EtcdStarter.java => CenterStarter.java} (98%) rename worker/src/main/java/com/jd/platform/jlog/worker/config/{EtcdConfig.java => ConfigCenter.java} (34%) delete mode 100644 worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java diff --git a/Dashboard/pom.xml b/Dashboard/pom.xml index 6e391f1..b921f02 100644 --- a/Dashboard/pom.xml +++ b/Dashboard/pom.xml @@ -55,11 +55,6 @@ zstd-jni 1.5.0-4 - - com.jd.ump - profiler - 6.2.13 - com.alibaba fastjson diff --git a/Dashboard/src/main/resources/logback-spring.xml b/Dashboard/src/main/resources/logback-spring.xml index dce10af..7202d5f 100644 --- a/Dashboard/src/main/resources/logback-spring.xml +++ b/Dashboard/src/main/resources/logback-spring.xml @@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息， --> - + diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index f12bafd..f6c4c06 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -1,11 +1,14 @@ package com.jd.platform.jlog.client; -import com.jd.platform.jlog.client.etcd.EtcdConfigFactory; -import com.jd.platform.jlog.client.etcd.EtcdStarter; +import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.udp.HttpSender; import com.jd.platform.jlog.client.udp.UdpClient; import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.common.config.ConfigCenterFactory; +import com.jd.platform.jlog.common.model.CenterConfig; +import com.jd.platform.jlog.common.model.TagConfig; +import com.jd.platform.jlog.common.model.TagHandler; /** * TracerClientStarter @@ -14,15 +17,18 @@ import com.jd.platform.jlog.client.udp.UdpSender; * @date 2021-08-13 */ public class TracerClientStarter { - /** - * etcd地址 - */ - private String etcdServer; /** * 机房 */ private Mdc mdc; + + private CenterConfig centerConfig; + + + private TagConfig tagConfig; + + /** * TracerClientStarter */ @@ -35,8 +41,9 @@ public class TracerClientStarter { public static class Builder { private String appName; - private String etcdServer; private Mdc mdc; + private CenterConfig centerConfig; + private TagConfig tagConfig; public Builder() { } @@ -51,16 +58,21 @@ public class TracerClientStarter { return this; } - public Builder setEtcdServer(String etcdServer) { - this.etcdServer = etcdServer; + public Builder setCenterConfig(CenterConfig centerConfig) { + this.centerConfig = centerConfig; + return this; + } + + public Builder setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; return this; } public TracerClientStarter build() { TracerClientStarter tracerClientStarter = new TracerClientStarter(appName); - tracerClientStarter.etcdServer = etcdServer; + tracerClientStarter.centerConfig = centerConfig; + tracerClientStarter.tagConfig = tagConfig; tracerClientStarter.mdc = mdc; - return tracerClientStarter; } } @@ -68,14 +80,15 @@ public class TracerClientStarter { /** * 启动监听etcd */ - public void startPipeline() { + public void startPipeline() throws Exception { //设置ConfigCenter - EtcdConfigFactory.buildConfigCenter(etcdServer); + ConfigCenterFactory.buildConfigCenter(centerConfig); + System.out.println("tagconfig"+tagConfig.getDelimiter()); + TagHandler.buildTag(tagConfig); Context.MDC = mdc; - EtcdStarter starter = new EtcdStarter(); - //与etcd相关的监听都开启 + Monitor starter = new Monitor(); starter.start(); UdpClient udpClient = new UdpClient(); diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java b/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java deleted file mode 100644 index 3e4b339..0000000 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.jd.platform.jlog.client.etcd; - - -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; -import com.jd.platform.jlog.common.config.zookeeper.ZkBuilder; - -/** - * @author wuweifeng wrote on 2020-01-07 - * @version 1.0 - */ -public class EtcdConfigFactory { - private static IConfigCenter configCenter; - - private EtcdConfigFactory() {} - - public static IConfigCenter configCenter() { - return configCenter; - } - - public static void buildConfigCenter(String etcdServer) { - //连接多个时，逗号分隔 - //configCenter = JdEtcdBuilder.build(etcdServer); - configCenter = ZkBuilder.build(etcdServer); - } -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java similarity index 97% rename from client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java rename to client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 595dae5..b8bd03f 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -20,14 +20,14 @@ import java.io.PrintWriter; import java.util.*; /** - * UserFilter + * HttpFilter * http://blog.chinaunix.net/uid-20783755-id-4729930.html * * @author wuweifeng * @version 1.0 * @date 2021-08-16 */ -public class UserFilter implements Filter { +public class HttpFilter implements Filter { /** * 获取切量百分比的 */ @@ -38,11 +38,11 @@ public class UserFilter implements Filter { /** * 传入百分比实现类 */ - public UserFilter(ITracerPercent iTracerPercent) { + public HttpFilter(ITracerPercent iTracerPercent) { this.iTracerPercent = iTracerPercent; } - public UserFilter() { + public HttpFilter() { iTracerPercent = new DefaultTracerPercentImpl(); } diff --git a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java similarity index 81% rename from client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java rename to client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java index 2aecf93..ea3f62a 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java @@ -1,10 +1,12 @@ -package com.jd.platform.jlog.client.etcd; +package com.jd.platform.jlog.client.task; import cn.hutool.core.collection.CollectionUtil; -import com.ibm.etcd.api.KeyValue; +import com.alibaba.nacos.api.exception.NacosException; import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.worker.WorkerInfoHolder; +import com.jd.platform.jlog.common.config.ConfigCenterEnum; +import com.jd.platform.jlog.common.config.ConfigCenterFactory; import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.constant.Constant; import io.grpc.StatusRuntimeException; @@ -18,13 +20,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** - * etcd连接管理器 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-08-13 + * @author tangbohu + * @version 1.0.0 + * @ClassName Watchdog.java + * @Description TODO + * @createTime 2022年02月12日 10:20:00 */ -public class EtcdStarter { +public class Monitor { /** * logger */ @@ -35,7 +37,7 @@ public class EtcdStarter { * 监听workerIp地址变化 */ public void start() { - fetchWorkerInfo(); + //fetchWorkerInfo(); } /** @@ -55,7 +57,7 @@ public class EtcdStarter { * 从配置中心获取worker的ip集合 */ private void fetch() { - IConfigCenter configCenter = EtcdConfigFactory.configCenter(); + IConfigCenter configCenter = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD); //获取所有worker的ip List keys = null; try { @@ -77,7 +79,12 @@ public class EtcdStarter { if (keys != null) { for (String key : keys) { //value里放的是ip地址 - String ipPort = configCenter.get(key); + String ipPort = null; + try { + ipPort = configCenter.get(key); + } catch (NacosException e) { + e.printStackTrace(); + } addresses.add(ipPort); } } diff --git a/clientdemo/pom.xml b/clientdemo/pom.xml index 052e0c4..24464dc 100644 --- a/clientdemo/pom.xml +++ b/clientdemo/pom.xml @@ -23,20 +23,25 @@ org.springframework.boot spring-boot-starter-web - + - + com.jd.platfrom.jlog client 1.4-SNAPSHOT - + org.springframework.boot + spring-boot-configuration-processor + true + + + diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java index fb1e00d..e73ba6d 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java @@ -1,5 +1,6 @@ package com.jd.platform.jlog.clientdemo; +import org.apache.catalina.filters.RequestFilter; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -14,6 +15,11 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; public class ClientDemoApplication { public static void main(String[] args) { - SpringApplication.run(ClientDemoApplication.class, args); + try { + SpringApplication.run(ClientDemoApplication.class, args); + }catch (Exception e){ + e.printStackTrace(); + } + } } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java index 5b4e423..a6a8549 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java @@ -1,12 +1,15 @@ package com.jd.platform.jlog.clientdemo.config; import com.jd.platform.jlog.client.TracerClientStarter; -import com.jd.platform.jlog.client.filter.UserFilter; -import org.springframework.beans.factory.annotation.Value; +import com.jd.platform.jlog.client.filter.HttpFilter; +import com.jd.platform.jlog.common.model.CenterConfig; +import com.jd.platform.jlog.common.model.TagConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - +import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; /** @@ -16,24 +19,48 @@ import javax.annotation.PostConstruct; * @version 1.0 * @date 2021-12-27 */ -@Configuration +@Component +@ConfigurationProperties(prefix = "jlog") public class DemoConfig { - @Value("${config.server}") - private String etcdServer; + private Logger logger = LoggerFactory.getLogger(getClass()); + + private CenterConfig centerConfig; + private TagConfig tagConfig ; + + public CenterConfig getCenterConfig() { + return centerConfig; + } + + public void setCenterConfig(CenterConfig centerConfig) { + this.centerConfig = centerConfig; + } + + public TagConfig getTagConfig() { + return tagConfig; + } + + public void setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; + } @PostConstruct - public void begin() { + public void begin() throws Exception { + TracerClientStarter tracerClientStarter = new TracerClientStarter.Builder() .setAppName("demo") - .setEtcdServer(etcdServer).build(); + .setCenterConfig(centerConfig) + .setTagConfig(tagConfig) + .build(); + logger.info("init centerConfig",centerConfig); + logger.info("init tagConfig",tagConfig); tracerClientStarter.startPipeline(); } @Bean public FilterRegistrationBean urlFilter() { FilterRegistrationBean registration = new FilterRegistrationBean(); - UserFilter userFilter = new UserFilter(); + HttpFilter userFilter = new HttpFilter(); registration.setFilter(userFilter); registration.addUrlPatterns("/*"); diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java new file mode 100644 index 0000000..87a9b6b --- /dev/null +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java @@ -0,0 +1,119 @@ +package com.jd.platform.jlog.clientdemo.config; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxy; +import ch.qos.logback.core.AppenderBase; +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.client.log.LogExceptionStackTrace; +import com.jd.platform.jlog.client.tracerholder.TracerHolder; +import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.common.model.RunLogMessage; +import com.jd.platform.jlog.common.model.TagHandler; +import org.slf4j.helpers.MessageFormatter; + +import java.util.Map; + +/** + * className：TracerLog4JAppender + * + * @author wuweifeng + * @version 1.0.0 + */ +public class TracerLogbackAppender extends AppenderBase { + /** + * 日志的堆栈级别，如直接在类里使用logger.info，则该值为0. 如调用另一个日志类如CommonLogger打印，则该值为1 + */ + private int loggerStage; + + @Override + protected void append(ILoggingEvent iLoggingEvent) { + try { + long tracerId = TracerHolder.getTracerId(); + /* if (0L == tracerId) { + return; + }*/ + RunLogMessage logMessage = getLogMessage(iLoggingEvent); + UdpSender.offerLogger(logMessage); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 转化为对象 + */ + private RunLogMessage getLogMessage(ILoggingEvent loggingEvent) { + RunLogMessage logMessage = new RunLogMessage(); + //设置链路唯一id + logMessage.setTracerId(TracerHolder.getTracerId()); + logMessage.setClassName(loggingEvent.getLoggerName()); + logMessage.setThreadName(loggingEvent.getThreadName()); + + StackTraceElement stackTraceElement = loggingEvent.getCallerData()[loggerStage]; + + String method = stackTraceElement.getMethodName(); + String line = String.valueOf(stackTraceElement.getLineNumber()); + logMessage.setMethodName(method + "(" + stackTraceElement.getFileName() + ":" + line + ")"); + logMessage.setLogLevel(loggingEvent.getLevel().toString()); + + logMessage.setCreateTime(loggingEvent.getTimeStamp()); + + String formattedMessage = getMessage(loggingEvent); + logMessage.setContent(formattedMessage); + Map map = TagHandler.extract(formattedMessage); + if(map != null){ + System.out.println("Logback tag map =>> "+ JSON.toJSONString(map)); + } + return logMessage; + } + + /** + * 日志正文信息 + */ + private String getMessage(ILoggingEvent logEvent) { + if (logEvent.getLevel() == Level.ERROR) { + if (logEvent.getThrowableProxy() != null) { + ThrowableProxy throwableProxy = (ThrowableProxy) logEvent.getThrowableProxy(); + String[] args = new String[]{logEvent.getFormattedMessage() + "\n" + LogExceptionStackTrace.erroStackTrace(throwableProxy.getThrowable()).toString()}; + return packageMessage("{}", args); + } else { + Object[] args = logEvent.getArgumentArray(); + if (args != null) { + for (int i = 0; i < args.length; i++) { + if (args[i] instanceof Throwable) { + args[i] = LogExceptionStackTrace.erroStackTrace(args[i]); + } + } + return packageMessage(logEvent.getMessage(), args); + } + } + } + return logEvent.getFormattedMessage(); + } + + private String packageMessage(String message, Object[] args) { + if (message != null && message.contains("{}")) { + return MessageFormatter.arrayFormat(message, args).getMessage(); + } + return packageMsg(message, args); + } + + private String packageMsg(String message, Object[] args) { + StringBuilder builder = new StringBuilder(128); + builder.append(message); + for (Object arg : args) { + builder.append("\n").append(arg); + } + return builder.toString(); + } + + + public int getLoggerStage() { + return loggerStage; + } + + public void setLoggerStage(int loggerStage) { + this.loggerStage = loggerStage; + } +} diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 60929ea..3a4a023 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -1,5 +1,9 @@ package com.jd.platform.jlog.clientdemo.web; +import com.alibaba.nacos.api.exception.NacosException; +import com.jd.platform.jlog.common.config.ConfigCenterEnum; +import com.jd.platform.jlog.common.config.ConfigCenterFactory; +import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.model.TracerBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,13 +26,28 @@ public class TestController { private static Logger RequestLog = LoggerFactory.getLogger("RequestLog"); @RequestMapping("/index") - public Object index() { + public Object index() throws NacosException { TracerBean tracerBean = new TracerBean(); tracerBean.setTracerId("11111"); + IConfigCenter client = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD); + try{ + client.put("/test","val1"); + }catch (Exception e){ + e.printStackTrace(); + } + + String val = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD).get("/test"); + System.out.println("val ===> "+val); RequestLog.info("哈哈哈哈哈哈"); return tracerBean; } + @RequestMapping("/log") + public Object log() throws NacosException { + RequestLog.info("|tag3=val3||tag4=val4||这是随便的log|"); + return 1; + } + } diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml index bf94e96..520bd91 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/clientdemo/src/main/resources/application.yml @@ -1,5 +1,22 @@ #etcd的地址，如有多个用逗号分隔 -config: - server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ +jlog: + center-config: + center: etcd + etcdServer: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ + nacosServer: 101.42.242.201:8848 + zkServer: 101.42.242.201:2181 + tag-config: + reqTags: + - tag1 + - tag2 + logTags: + - tag3 + - tag4 + regex: "" + delimiter: "|" + join: = + + + server: port: 8081 \ No newline at end of file diff --git a/clientdemo/src/main/resources/log4j2.xml b/clientdemo/src/main/resources/log4j2.xml deleted file mode 100644 index 00f41b9..0000000 --- a/clientdemo/src/main/resources/log4j2.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/clientdemo/src/main/resources/logback-spring.xml b/clientdemo/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..22536e3 --- /dev/null +++ b/clientdemo/src/main/resources/logback-spring.xml @@ -0,0 +1,62 @@ + + + logs + + + + + + + + + + + + ${log.colorPattern} + + + + + + + + + + ${log.path}/info/info.%d{yyyy-MM-dd}.log + ${log.maxHistory} + + + ${log.pattern} + + + INFO + ACCEPT + DENY + + + + + + ${log.path}/error/error.%d{yyyy-MM-dd}.log + + + ${log.pattern} + + + ERROR + ACCEPT + DENY + + + + + + + + + + + + + + diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java b/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java index e2a8343..140b2d5 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java +++ b/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java @@ -7,10 +7,6 @@ import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IpUtils; import com.jd.platform.jlog.common.utils.ZstdUtils; import org.apache.tomcat.util.codec.binary.Base64; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; import java.nio.charset.StandardCharsets; import java.util.*; @@ -22,11 +18,11 @@ import java.util.*; * @version 1.0 * @date 2021-12-27 */ -@SpringBootTest(classes = ClientDemoApplication.class) -@RunWith(SpringRunner.class) +//@SpringBootTest(classes = ClientDemoApplication.class) +//@RunWith(SpringRunner.class) public class TracerPacketTest { - @Test + //@Test public void testSendUdp() { TracerBean tracerBean = new TracerBean(); List> tracerObject = new ArrayList<>(); diff --git a/common/pom.xml b/common/pom.xml index 73d8387..aa3005d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -104,6 +104,22 @@ guava ${caffeine.version} + + + com.alibaba.nacos + nacos-client + + 1.4.3 + + + + + com.ctrip.framework.apollo + apollo-client + 1.9.2 + + + \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java new file mode 100644 index 0000000..e6120cc --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.common.config; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigCenterEnum.java + * @Description TODO + * @createTime 2022年02月10日 20:32:00 + */ +public enum ConfigCenterEnum { + APOLLO, + NACOS, + ETCD, + ZK, +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java new file mode 100644 index 0000000..6a7e4c3 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java @@ -0,0 +1,58 @@ +package com.jd.platform.jlog.common.config; + + +import com.google.common.collect.Maps; +import com.jd.platform.jlog.common.config.apollo.ApolloClient; +import com.jd.platform.jlog.common.config.etcd.JdEtcdClient; +import com.jd.platform.jlog.common.config.nacos.NacosClient; +import com.jd.platform.jlog.common.config.zookeeper.ZKClient; +import com.jd.platform.jlog.common.model.CenterConfig; +import com.sun.deploy.util.ReflectionUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.getCenter; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigCenterFactory.java + * @Description TODO + * @createTime 2022年02月10日 20:31:00 + */ +public class ConfigCenterFactory { + + private static Map configCenterMap = Maps.newHashMap(); + + + public static void buildConfigCenter(CenterConfig config) throws Exception { + + ConfigCenterEnum center = getCenter(config); + + switch (center){ + case APOLLO: + configCenterMap.put(center, new ApolloClient().buildClient(config)); + break; + case NACOS: + configCenterMap.put(center,new NacosClient().buildClient(config)); + break; + case ETCD: + configCenterMap.put(center, new JdEtcdClient().buildClient(config)); + break; + case ZK: + configCenterMap.put(center, new ZKClient().buildClient(config)); + break; + default: + System.out.println("Unsupported config center"); + } + } + + + public static IConfigCenter getClient(ConfigCenterEnum center) { + return configCenterMap.get(center); + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java index 8ebd4d3..1ef1333 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java @@ -1,88 +1,49 @@ package com.jd.platform.jlog.common.config; -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.client.kv.KvClient; +import com.alibaba.nacos.api.exception.NacosException; +import com.jd.platform.jlog.common.model.CenterConfig; import java.util.List; +import java.util.Map; /** - * 配合中心接口 - * - * @author wuweifeng wrote on 2019-12-09 - * @version 1.0 + * @author tangbohu + * @version 1.0.0 + * @ClassName AbsConfigCenter.java + * @Description TODO + * @createTime 2022年02月10日 20:33:00 */ public interface IConfigCenter { - /** - * 存入key，value - */ - void put(String key, String value); - /** - * 存入key、value，和租约id - */ - void put(String key, String value, long leaseId); - - /** - * 删除一个lease - */ - void revoke(long leaseId); /** - * 存入key、value，和过期时间，单位是秒 + * 构建config-center client + * @param config + * @throws Exception */ - long putAndGrant(String key, String value, long ttl); + IConfigCenter buildClient(CenterConfig config) throws Exception; /** - * 给key设置新的leaseId + * 存入key，value */ - long setLease(String key, long leaseId); - - void delete(String key); + void put(String key, String value); /** * 根据key，获取value */ - String get(String key); + String get(String key) throws NacosException; - /** - * 获取指定前缀的所有key - */ - List getPrefixKey(String key); - /** - * 监听key - */ - KvClient.WatchIterator watch(String key); - - /** - * 监听前缀为key的 - */ - KvClient.WatchIterator watchPrefix(String key); - - /** - * 自动续约 - * @param frequencySecs 续约频率，最小是4秒，默认是5秒 - * @param minTtl 最小存活时间，最小是2秒，默认是10秒 - * @return 返回leaseId - */ - long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception; /** - * 构建一个可自动续约的lease + * 删除 + * @param key */ - long buildAliveLease(int frequencySecs, int minTtl) throws Exception; + void delete(String key); - /** - * 构建一个普通lease - */ - long buildNormalLease(long ttl); /** - * 判断剩余的过期时间 + * 获取指定前缀的所有key */ - long timeToLive(long leaseId); + List getPrefixKey(String key); - /** - * 根据key，获取value - */ - KeyValue getKv(String key); } diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java new file mode 100644 index 0000000..86f9818 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java @@ -0,0 +1,57 @@ +package com.jd.platform.jlog.common.config.apollo; + +import com.ctrip.framework.apollo.Config; +import com.ctrip.framework.apollo.ConfigService; +import com.jd.platform.jlog.common.config.IConfigCenter; +import com.jd.platform.jlog.common.model.CenterConfig; + +import java.util.List; +import java.util.Map; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloClient.java + * @Description TODO + * @createTime 2022年02月10日 20:37:00 + */ + + +public class ApolloClient implements IConfigCenter { + + private Config config; + + public ApolloClient() {} + + public ApolloClient(Config config) { + this.config = config; + } + + @Override + public IConfigCenter buildClient(CenterConfig centerConfig){ + config = ConfigService.getAppConfig(); + return new ApolloClient(config); + } + + @Override + public void put(String key, String value) { + + } + + @Override + public String get(String key) { + return null; + } + + @Override + public void delete(String key) { + + } + + @Override + public List getPrefixKey(String key) { + return null; + } + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java deleted file mode 100644 index b9fef7e..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.jlog.common.config.etcd; - -import com.ibm.etcd.client.EtcdClient; - -/** - * etcd连接器 - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class JdEtcdBuilder { - - /** - * @param endPoints 如https://127.0.0.1:2379 有多个时逗号分隔 - */ - public static JdEtcdClient build(String endPoints) { - return new JdEtcdClient(EtcdClient.forEndpoints(endPoints).withPlainText().build()); - } -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java index e274685..9dbac30 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java @@ -3,21 +3,17 @@ package com.jd.platform.jlog.common.config.etcd; import cn.hutool.core.collection.CollectionUtil; import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.LeaseGrantResponse; import com.ibm.etcd.api.RangeResponse; +import com.ibm.etcd.client.EtcdClient; import com.ibm.etcd.client.KvStoreClient; import com.ibm.etcd.client.kv.KvClient; import com.ibm.etcd.client.lease.LeaseClient; -import com.ibm.etcd.client.lease.PersistentLease; import com.ibm.etcd.client.lock.LockClient; import com.jd.platform.jlog.common.config.IConfigCenter; +import com.jd.platform.jlog.common.model.CenterConfig; -import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import static java.util.concurrent.TimeUnit.SECONDS; +import java.util.Map; /** * etcd客户端 @@ -27,10 +23,12 @@ import static java.util.concurrent.TimeUnit.SECONDS; */ public class JdEtcdClient implements IConfigCenter { + private KvClient kvClient; private LeaseClient leaseClient; private LockClient lockClient; + public JdEtcdClient() {} public JdEtcdClient(KvStoreClient kvStoreClient) { this.kvClient = kvStoreClient.getKvClient(); @@ -39,28 +37,10 @@ public class JdEtcdClient implements IConfigCenter { } - public LeaseClient getLeaseClient() { - return leaseClient; - } - - public void setLeaseClient(LeaseClient leaseClient) { - this.leaseClient = leaseClient; - } - - public KvClient getKvClient() { - return kvClient; - } - - public void setKvClient(KvClient kvClient) { - this.kvClient = kvClient; - } - - public LockClient getLockClient() { - return lockClient; - } - - public void setLockClient(LockClient lockClient) { - this.lockClient = lockClient; + @Override + public IConfigCenter buildClient(CenterConfig config){ + String etcdServer = config.getEtcdServer(); + return new JdEtcdClient(EtcdClient.forEndpoints(etcdServer).withPlainText().build()); } @Override @@ -68,39 +48,10 @@ public class JdEtcdClient implements IConfigCenter { kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value)).sync(); } - @Override - public void put(String key, String value, long leaseId) { - kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value), leaseId).sync(); - } - - @Override - public void revoke(long leaseId) { - leaseClient.revoke(leaseId); - } - - @Override - public long putAndGrant(String key, String value, long ttl) { - LeaseGrantResponse lease = leaseClient.grant(ttl).sync(); - put(key, value, lease.getID()); - return lease.getID(); - } - - @Override - public long setLease(String key, long leaseId) { - kvClient.setLease(ByteString.copyFromUtf8(key), leaseId); - return leaseId; - } - - @Override - public void delete(String key) { - kvClient.delete(ByteString.copyFromUtf8(key)).sync(); - } - @Override public String get(String key) { RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync(); List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { return null; } @@ -108,68 +59,13 @@ public class JdEtcdClient implements IConfigCenter { } @Override - public KeyValue getKv(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync(); - List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { - return null; - } - return keyValues.get(0); - } - -// @Override -// public List getPrefix(String key) { -// RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); -// return rangeResponse.getKvsList(); -// } - - @Override - public List getPrefixKey(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync(); - return rangeResponse.getKvsList().stream().map( - kv -> kv.getKey().toStringUtf8() - ).collect(Collectors.toList()); - } - - @Override - public KvClient.WatchIterator watch(String key) { - return kvClient.watch(ByteString.copyFromUtf8(key)).start(); - } - - @Override - public KvClient.WatchIterator watchPrefix(String key) { - return kvClient.watch(ByteString.copyFromUtf8(key)).asPrefix().start(); - } - - @Override - public long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception { - //minTtl秒租期，每frequencySecs秒续约一下 - PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start(); - long newId = lease.get(3L, SECONDS); - put(key, value, newId); - return newId; - } - - @Override - public long buildAliveLease(int frequencySecs, int minTtl) throws Exception { - PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start(); + public void delete(String key) { - return lease.get(3L, SECONDS); } @Override - public long buildNormalLease(long ttl) { - LeaseGrantResponse lease = leaseClient.grant(ttl).sync(); - return lease.getID(); + public List getPrefixKey(String key) { + return null; } - @Override - public long timeToLive(long leaseId) { - try { - return leaseClient.ttl(leaseId).get().getTTL(); - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); - return 0L; - } - } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java new file mode 100644 index 0000000..e970fd5 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java @@ -0,0 +1,68 @@ +package com.jd.platform.jlog.common.config.nacos; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; +import com.jd.platform.jlog.common.config.IConfigCenter; +import com.jd.platform.jlog.common.model.CenterConfig; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosClient.java + * @Description TODO + * @createTime 2022年02月10日 21:47:00 + */ +public class NacosClient implements IConfigCenter { + + + private ConfigService configService; + + public NacosClient(){} + + public NacosClient(ConfigService configService){ + this.configService = configService; + } + + + @Override + public IConfigCenter buildClient(CenterConfig config) throws NacosException { + String serverAddr = config.getNacosServer(); + Properties properties = new Properties(); + properties.put("serverAddr", serverAddr); + ConfigService configService = NacosFactory.createConfigService(properties); + return new NacosClient(configService); + } + + @Override + public void put(String key, String value){ + try { + boolean result = configService.publishConfig("data1", "group1", "nacosContentTEXT"); + System.out.println("publishConfig result==-> "+result); + } catch (Exception e) { + System.out.println("==-> "+e.toString()); + e.printStackTrace(); + } + } + + @Override + public String get(String key) throws NacosException { + return configService.getConfig("data1", "group1", 5000); + } + + @Override + public void delete(String key) { + + } + + @Override + public List getPrefixKey(String key) { + return null; + } + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java index c257bfe..4bed987 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java +++ b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java @@ -1,9 +1,11 @@ package com.jd.platform.jlog.common.config.zookeeper; -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.client.kv.KvClient; + import com.jd.platform.jlog.common.config.IConfigCenter; +import com.jd.platform.jlog.common.model.CenterConfig; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs; @@ -18,10 +20,28 @@ public class ZKClient implements IConfigCenter { private CuratorFramework curator; + public ZKClient() {} + public ZKClient(CuratorFramework curatorFramework) { this.curator = curatorFramework; } + + @Override + public IConfigCenter buildClient(CenterConfig config) { + String zkServer = config.getZkServer(); + CuratorFramework client = CuratorFrameworkFactory.builder().connectString(zkServer) + // 连接超时时间 + .sessionTimeoutMs(10000) + // 会话超时时间 + .connectionTimeoutMs(10000) + // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + client.start(); + return new ZKClient(client); + } + @Override public void put(String key, String value) { try { @@ -35,35 +55,16 @@ public class ZKClient implements IConfigCenter { } - @Override - public void put(String key, String value, long leaseId) { - } @Override - public void revoke(long leaseId) { - - } - - @Override - public long putAndGrant(String key, String value, long ttl) { - return 0; - } - - @Override - public long setLease(String key, long leaseId) { - return 0; - } - - @Override - public void delete(String key) { - - } + public void delete(String key) { } @Override public String get(String key) { try { - return curator.getData().forPath(key).toString(); + byte[] bt = curator.getData().forPath(key); + return new String(bt, "utf-8"); } catch (Exception e) { e.printStackTrace(); } @@ -81,38 +82,5 @@ public class ZKClient implements IConfigCenter { return null; } - @Override - public KvClient.WatchIterator watch(String key) { - return null; - } - - @Override - public KvClient.WatchIterator watchPrefix(String key) { - return null; - } - @Override - public long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception { - return 0; - } - - @Override - public long buildAliveLease(int frequencySecs, int minTtl) throws Exception { - return 0; - } - - @Override - public long buildNormalLease(long ttl) { - return 0; - } - - @Override - public long timeToLive(long leaseId) { - return 0; - } - - @Override - public KeyValue getKv(String key) { - return null; - } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java deleted file mode 100644 index a5772fb..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZkBuilder.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.jd.platform.jlog.common.config.zookeeper; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; - -/** - * @author shenkaiwen5 - * @version 1.0 - * @date 2022-01-06 - */ -public class ZkBuilder { - /** - * 构建ZKClient - */ - public static ZKClient build(String endPoints) { - CuratorFramework client = CuratorFrameworkFactory.builder().connectString(endPoints) - // 连接超时时间 - .sessionTimeoutMs(1000) - // 会话超时时间 - .connectionTimeoutMs(1000) - // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build(); - client.start(); - - return new ZKClient(client); - } -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java new file mode 100644 index 0000000..95d453a --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java @@ -0,0 +1,19 @@ +package com.jd.platform.jlog.common.config.zookeeper; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +public class ZookeeperTest { + public static void main(String[] args) throws Exception { + ZooKeeper zk = new ZooKeeper("101.42.242.201:2181", 3000, null); + + System.out.println(zk.getClass()); + if(zk.exists("/test22", false) == null) + { + zk.create("/test22", "znode1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + System.out.println(new String(zk.getData("/test22", false, null))); + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index 6383e21..cc6e197 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -1,5 +1,11 @@ package com.jd.platform.jlog.common.constant; +import com.alibaba.fastjson.JSON; +import com.alibaba.nacos.common.utils.StringUtils; + +import java.util.HashSet; +import java.util.Set; + /** * 常量工具类Constant * @@ -24,4 +30,29 @@ public class Constant { * 当客户端要删除某个key时，就往etcd里赋值这个value，设置1秒过期，就算删除了 */ public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; + + + + public static int TAG_NORMAL_KEY_MAX_LEN = 20; + + public static String TAG_NORMAL_KEY = "normal"; + + public static final Set SPECIAL_CHAR = new HashSet() { + { + add("*"); + add("."); + add("?"); + add("+"); + add("$"); + add("^"); + add("["); + add("]"); + add("("); + add(")"); + add("{"); + add("}"); + add("|"); + } + }; + } diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java b/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java new file mode 100644 index 0000000..7280215 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java @@ -0,0 +1,55 @@ +package com.jd.platform.jlog.common.model; + +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.config.ConfigCenterEnum; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName JConfig.java + * @Description TODO + * @createTime 2022年02月11日 23:11:00 + */ +public class CenterConfig { + + private String etcdServer; + + private String nacosServer; + + private String zkServer; + + + public String getEtcdServer() { + return etcdServer; + } + + public void setEtcdServer(String etcdServer) { + this.etcdServer = etcdServer; + } + + public String getNacosServer() { + return nacosServer; + } + + public void setNacosServer(String nacosServer) { + this.nacosServer = nacosServer; + } + + public String getZkServer() { + return zkServer; + } + + public void setZkServer(String zkServer) { + this.zkServer = zkServer; + } + + + @Override + public String toString() { + return "CenterConfig{" + + ", etcdServer='" + etcdServer + '\'' + + ", nacosServer='" + nacosServer + '\'' + + ", zkServer='" + zkServer + '\'' + + '}'; + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java new file mode 100644 index 0000000..7887eca --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java @@ -0,0 +1,126 @@ +package com.jd.platform.jlog.common.model; + +import java.io.Serializable; +import java.util.List; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName TagConfig.java + * @Description TODO + * @createTime 2022年02月13日 22:35:00 + */ +public class TagConfig implements Serializable { + + private List reqTags; + + private List logTags; + + private String regex; + + private String delimiter = "|"; + + private String join = "="; + + public List getReqTags() { + return reqTags; + } + + public void setReqTags(List reqTags) { + this.reqTags = reqTags; + } + + public List getLogTags() { + return logTags; + } + + public void setLogTags(List logTags) { + this.logTags = logTags; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public String getJoin() { + return join; + } + + public void setJoin(String join) { + this.join = join; + } + + + public static final class Builder { + private List reqTags; + private List logTags; + private String regex; + private String delimiter = "|"; + private String join = "="; + + public Builder() { + } + + public static Builder aTagConfig() { + return new Builder(); + } + + public Builder reqTags(List reqTags) { + this.reqTags = reqTags; + return this; + } + + public Builder logTags(List logTags) { + this.logTags = logTags; + return this; + } + + public Builder regex(String regex) { + this.regex = regex; + return this; + } + + public Builder delimiter(String delimiter) { + this.delimiter = delimiter; + return this; + } + + public Builder join(String join) { + this.join = join; + return this; + } + + public TagConfig build() { + TagConfig tagConfig = new TagConfig(); + tagConfig.setReqTags(reqTags); + tagConfig.setLogTags(logTags); + tagConfig.setRegex(regex); + tagConfig.setDelimiter(delimiter); + tagConfig.setJoin(join); + return tagConfig; + } + } + + @Override + public String toString() { + return "TagConfig{" + + "reqTags=" + reqTags + + ", logTags=" + logTags + + ", regex='" + regex + '\'' + + ", delimiter='" + delimiter + '\'' + + ", join='" + join + '\'' + + '}'; + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java new file mode 100644 index 0000000..5d333b3 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java @@ -0,0 +1,178 @@ +package com.jd.platform.jlog.common.model; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.ConfigUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY; +import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY_MAX_LEN; +import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Tag.java + * @Description TODO + * @createTime 2022年02月12日 21:28:00 + */ +public class TagHandler { + + private static Logger logger = LoggerFactory.getLogger(TagHandler.class); + + private Set reqTags; + + private Set logTags; + + private String regex; + + private String delimiter = "|"; + + private int delimiterLen = delimiter.length(); + + private String join = "="; + + private Pattern pattern; + + private static TagHandler INSTANCE = null; + + public static void buildTag(TagConfig tagConfig) { + String regex = tagConfig.getRegex(); + TagHandler handler = new TagHandler(); + handler.reqTags = new HashSet<>(tagConfig.getReqTags()); + handler.logTags = new HashSet<>(tagConfig.getLogTags()); + if(regex != null && !"".equals(regex)){ + handler.regex = regex; + handler.pattern = Pattern.compile(regex); + }else{ + handler.delimiter = tagConfig.getDelimiter(); + handler.delimiterLen = tagConfig.getDelimiter().length(); + handler.join = tagConfig.getJoin(); + System.out.println("delimiter==> "+tagConfig.getDelimiter()); + + String escapeDelimiter = ConfigUtil.escapeExprSpecialWord(tagConfig.getDelimiter()); + System.out.println("## escapeDelimiter==> "+escapeDelimiter); + + // String str = "("+ formatDelimiter +"[\\w\\W]*?"+ formatDelimiter +")"; + String str = String.format("(%s[\\w\\W]*?%s)", escapeDelimiter, escapeDelimiter); + System.out.println("这里 实力话了"+str); + + handler.pattern = Pattern.compile(str); + } + INSTANCE = handler; + } + + + + + public static Map extract(String content) { + if(INSTANCE == null){ + return null; + } + + Map tagMap = new HashMap<>(3); + Matcher m = INSTANCE.pattern.matcher(content); + System.out.println("content==> "+content); + + while (m.find()) { + String str = m.group().substring(INSTANCE.delimiterLen, m.group().length() - INSTANCE.delimiterLen); + if(str.contains(INSTANCE.join)){ + String[] arr = str.split(INSTANCE.join); + System.out.println("INSTANCE.logTags==> "+INSTANCE.logTags); + if(INSTANCE.logTags.contains(arr[0])){ + tagMap.put(arr[0], arr[1]); + } + }else if(str.length() < TAG_NORMAL_KEY_MAX_LEN){ + System.out.println("INSTANCE.TAG_NORMAL_KEY==> "+str); + + tagMap.put(TAG_NORMAL_KEY, str); + }else{ + if (RANDOM.nextInt(50) == 1) { + logger.info("Some logs lack tags and are larger than 20 in length. Therefore, they are simply stored"); + } + } + } + return tagMap; + } + + + + + // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); + + public static List extractTest(String content) { + + List list = new ArrayList<>(); + Matcher m = BRACKET_PATTERN.matcher(content); + while (m.find()) { + list.add(m.group().substring(1, m.group().length() - 1)); + // list.add(m.group().substring(2, m.group().length() - 2)); + // list.add(m.group().substring(3, m.group().length()-3)); + } + return list; + } + + + //static Pattern BRACKET_PATTERN = Pattern.compile("(\\{\\{[\\w\\W]*?\\}\\})"); + // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|\\|[\\w\\W]*?\\|\\|)"); + static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); + + static String str1 = "|订单完成|订单类型=1"; + static String str2 = " ddddd"; + + // String reg=".*ll.*"; //判断字符串中是否含有ll + // System.out.println(str.matches(reg)); + + /* static String str1 = "||a=1||b=2||qwewe"; + static String str2 = "||a=1||b=2||qwewe||"; + static String str3 = "||a=1||eee||b=2"; + static String str4 = "||a=1||eee||b=2||";*/ + + + // static String str3 = "|a=1|这是普通log|b=2"; + // static String str4 = "|a=1|这是普通log|b=2|"; + + public static void main(String[] args) { + // System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1))); + System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2))); + // System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3))); + // System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4))); + } + + + + + /* Map requestMap = new HashMap<>(16); + requestMap.put("tracerId", "1"); + requestMap.put("pin", UUID.randomUUID()); + requestMap.put("appName", "myTest"); + requestMap.put("uuid", "uuid" + UUID.randomUUID()); + requestMap.put("client", "android"); + requestMap.put("clientVersion", "10.3.2"); + requestMap.put("ip", "127.0.0.1"); + requestMap.put("serverIp", "127.0.0.1"); + requestMap.put("uri", "test"); + tracerObject.add(requestMap); + + + + /* @Bean + public IConfigCenter client() throws Exception { + CenterConfig jConfig = new CenterConfig.Builder() + .etcdServer(etcdServer) + .nacosServer(nacosServer) + .zkServer(zkServer) + .build(); + ConfigCenterFactory.buildConfigCenter(jConfig); + logger.info("jConfig : " + jConfig); + //连接多个时，逗号分隔 + //return JdEtcdBuilder.build(etcdServer); + return ConfigCenterFactory.getClient(jConfig.getCenter()); + }*/ + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java new file mode 100644 index 0000000..e42a753 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -0,0 +1,67 @@ +package com.jd.platform.jlog.common.utils; + +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.config.ConfigCenterEnum; +import com.jd.platform.jlog.common.constant.Constant; +import com.jd.platform.jlog.common.model.CenterConfig; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigUtil.java + * @Description TODO + * @createTime 2022年02月13日 21:40:00 + */ +public class ConfigUtil { + + + private static final String SERVER_SUFFIX = "Server"; + + public static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); + + + + public static String escapeExprSpecialWord(String str) { + + if(str != null && str.length() > 0){ + for (String s : Constant.SPECIAL_CHAR) { + if (str.contains(s)) { + str = str.replace(s, "\\" + s); + } + } + } + return str; + } + + + public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception { + Class clz = config.getClass(); + Field[] fields = clz.getDeclaredFields(); + + for (Field field : fields) { + Method m = clz.getMethod("get" + getMethodName(field.getName())); + String val = (String)m.invoke(config); + if (val != null) { + for (ConfigCenterEnum center : ConfigCenterEnum.values()) { + String fd = field.getName().replace(SERVER_SUFFIX, ""); + if(center.name().equals(fd.toUpperCase())){ + return center; + } + } + } + } + throw new Exception("Configuration center cannot be found"); + } + + + private static String getMethodName(String fieldName) { + byte[] items = fieldName.getBytes(); + items[0] = (byte) ((char) items[0] - 'a' + 'A'); + return new String(items); + } +} diff --git a/worker/pom.xml b/worker/pom.xml index 771b8f6..d1a06a8 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -43,6 +43,12 @@ disruptor 3.4.1 + + + org.springframework.boot + spring-boot-configuration-processor + true + diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java b/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java index 7c76876..08ff060 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/Starter.java @@ -1,6 +1,6 @@ package com.jd.platform.jlog.worker; -import com.jd.platform.jlog.worker.config.EtcdStarter; +import com.jd.platform.jlog.worker.config.CenterStarter; import com.jd.platform.jlog.worker.store.TracerLogToDbStore; import com.jd.platform.jlog.worker.store.TracerModelToDbStore; import com.jd.platform.jlog.worker.udp.UdpServer; @@ -42,7 +42,7 @@ public class Starter { * etcd启动器 */ @Resource - private EtcdStarter etcdStarter; + private CenterStarter centerStarter; @PostConstruct public void start() { @@ -58,7 +58,7 @@ public class Starter { tracerLogToDbStore.beginIntoDb(); //上报自己ip到配置中心 - etcdStarter.uploadSelfInfo(); + centerStarter.uploadSelfInfo(); } } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java similarity index 98% rename from worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java rename to worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java index 5a514a0..8835f51 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java @@ -19,7 +19,7 @@ import java.util.concurrent.TimeUnit; * @date 2021-08-12 */ @Component -public class EtcdStarter { +public class CenterStarter { /** * 该worker为哪个app服务 */ diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java similarity index 34% rename from worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java rename to worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java index 3c8a7f1..3e7d533 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java @@ -1,13 +1,18 @@ package com.jd.platform.jlog.worker.config; +import com.jd.platform.jlog.common.config.ConfigCenterEnum; +import com.jd.platform.jlog.common.config.ConfigCenterFactory; import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder; -import com.jd.platform.jlog.common.config.zookeeper.ZkBuilder; +import com.jd.platform.jlog.common.model.CenterConfig; +import com.jd.platform.jlog.common.model.TagConfig; +import com.jd.platform.jlog.common.utils.ConfigUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; /** @@ -15,19 +20,38 @@ import org.springframework.context.annotation.Configuration; * @author wuweifeng wrote on 2019-12-06 * @version 1.0 */ -@Configuration -public class EtcdConfig { - @Value("${config.server}") - private String etcdServer; +@Component +@ConfigurationProperties(prefix = "jlog") +public class ConfigCenter { private Logger logger = LoggerFactory.getLogger(getClass()); + private CenterConfig centerConfig; + private TagConfig tagConfig ; + + public CenterConfig getCenterConfig() { + return centerConfig; + } + + public void setCenterConfig(CenterConfig centerConfig) { + this.centerConfig = centerConfig; + } + + public TagConfig getTagConfig() { + return tagConfig; + } + + public void setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; + } + @Bean - public IConfigCenter client() { - logger.info("etcd address : " + etcdServer); + public IConfigCenter client() throws Exception { + + ConfigCenterFactory.buildConfigCenter(centerConfig); //连接多个时，逗号分隔 //return JdEtcdBuilder.build(etcdServer); - return ZkBuilder.build(etcdServer); + return ConfigCenterFactory.getClient(ConfigUtil.getCenter(centerConfig)); } } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java deleted file mode 100644 index 064182a..0000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.jd.platform.jlog.worker.config; - -/** - * 配置中心接口 - * @author wuweifeng - * @version 1.0 - * @date 2021-08-13 - */ -public interface IConfig { -} diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 019ac48..c0030ca 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -6,7 +6,7 @@ queue: maxSize: ${queueSize:16384} preDbSize: ${preDbSize:10000} #etcd的地址，如有多个用逗号分隔 -config: +jConfig: mdc: ${mdc:default} server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ workerPath: ${workerPath:default} #该worker放到哪个path下，譬如放/app1下，则该worker只能被app1使用，不会为其他client提供服务 -- Gitee From c6a7fd038364f967c368aa1ff8070250124e06f2 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Tue, 22 Feb 2022 19:28:09 +0800 Subject: [PATCH 09/22] =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=99=A8=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 6 + .../jlog/client/TracerClientStarter.java | 71 +++-- .../jlog/client/filter/HttpFilter.java | 17 +- .../jd/platform/jlog/client/task/Monitor.java | 60 ++--- .../platform/jlog/client/udp/UdpSender.java | 7 +- .../jlog/client/worker/WorkerInfoHolder.java | 3 +- .../clientdemo/ClientDemoApplication.java | 2 - .../TracerLogbackAppender.java | 8 +- .../jlog/clientdemo/config/DemoConfig.java | 16 +- .../jlog/clientdemo/web/TestController.java | 30 ++- .../platform/jlog/clientdemo/web/TestReq.java | 15 ++ .../src/main/resources/application.properties | 1 + clientdemo/src/main/resources/application.yml | 15 +- .../src/main/resources/application.ymlbak | 22 ++ .../src/main/resources/logback-spring.xml | 2 +- common/pom.xml | 79 +----- .../common/config/apollo/ApolloClient.java | 57 ---- .../common/config/zookeeper/ZKClient.java | 86 ------- .../config/zookeeper/ZookeeperTest.java | 19 -- .../jlog/common/constant/Constant.java | 3 - .../jlog/common/model/CenterConfig.java | 55 ---- .../jlog/common/{model => tag}/TagConfig.java | 3 +- .../common/{model => tag}/TagHandler.java | 155 +++++------ .../platform/jlog/common/utils/AsyncPool.java | 5 + .../jlog/common/utils/AsyncWorker.java | 37 +++ .../jlog/common/utils/CollectionUtil.java | 84 ++++++ .../jlog/common/utils/ConfigUtil.java | 9 +- .../platform/jlog/common/utils/IdWorker.java | 6 +- .../jlog/common/utils/StringUtil.java | 45 ++++ config/config-apollo/pom.xml | 42 +++ .../config/apollo/ApolloConfigurator.java | 172 +++++++++++++ .../apollo/ApolloConfiguratorProvider.java | 15 ++ .../jlog/config/apollo/ApolloConstant.java | 22 ++ .../jlog/config/apollo/ApolloListener.java | 21 ++ ...jd.platform.jlog.core.ConfiguratorProvider | 1 + config/config-core/pom.xml | 21 ++ .../platform/jlog/core/ConfigChangeEvent.java | 139 ++++++++++ .../jlog/core/ConfigChangeListener.java | 65 +++++ .../platform/jlog/core/ConfigChangeType.java | 11 + .../jd/platform/jlog/core/Configurator.java | 90 +++++++ .../jlog/core/ConfiguratorFactory.java | 75 ++++++ .../jlog/core/ConfiguratorProvider.java | 14 + .../platform/jlog/core/FileConfigurator.java | 242 +++++++++++++++++ config/config-etcd/pom.xml | 26 ++ .../jlog/etcd/EtcdConfigurationProvider.java | 12 + .../platform/jlog/etcd/EtcdConfigurator.java | 137 ++++++++++ .../jd/platform/jlog/etcd/EtcdListener.java | 60 +++++ ...jd.platform.jlog.core.ConfiguratorProvider | 1 + config/config-nacos/pom.xml | 27 ++ .../jlog/nacos/NacosConfigurator.java | 243 ++++++++++++++++++ .../jlog/nacos/NacosConfiguratorProvider.java | 12 + .../jd/platform/jlog/nacos/NacosConstant.java | 20 ++ .../jd/platform/jlog/nacos/NacosListener.java | 86 +++++++ ...jd.platform.jlog.core.ConfiguratorProvider | 1 + config/config-zk/pom.xml | 41 +++ .../jd/platform/jlog/zk/ZkConfigurator.java | 194 ++++++++++++++ .../jlog/zk/ZkConfiguratorProvider.java | 17 ++ .../com/jd/platform/jlog/zk/ZkConstant.java | 18 ++ .../com/jd/platform/jlog/zk/ZkListener.java | 93 +++++++ .../java/com/jd/platform/jlog/zk/ZkTest.java | 63 +++++ ...jd.platform.jlog.core.ConfiguratorProvider | 1 + config/pom.xml | 36 +++ pom.xml | 1 + worker/pom.xml | 5 + .../jlog/worker/config/CenterStarter.java | 14 +- .../jlog/worker/config/ConfigCenter.java | 29 +-- .../worker/controller/UserController.java | 5 +- .../jlog/worker/disruptor/TracerConsumer.java | 7 +- .../jlog/worker/store/TracerLogToDbStore.java | 8 +- .../worker/store/TracerModelToDbStore.java | 2 +- 70 files changed, 2466 insertions(+), 541 deletions(-) rename clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/{config => appender}/TracerLogbackAppender.java (95%) create mode 100644 clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java create mode 100644 clientdemo/src/main/resources/application.properties create mode 100644 clientdemo/src/main/resources/application.ymlbak delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java rename common/src/main/java/com/jd/platform/jlog/common/{model => tag}/TagConfig.java (97%) rename common/src/main/java/com/jd/platform/jlog/common/{model => tag}/TagHandler.java (50%) create mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java create mode 100644 config/config-apollo/pom.xml create mode 100644 config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java create mode 100644 config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java create mode 100644 config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java create mode 100644 config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java create mode 100644 config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider create mode 100644 config/config-core/pom.xml create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java create mode 100644 config/config-etcd/pom.xml create mode 100644 config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java create mode 100644 config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java create mode 100644 config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java create mode 100644 config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider create mode 100644 config/config-nacos/pom.xml create mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java create mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java create mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java create mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java create mode 100644 config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider create mode 100644 config/config-zk/pom.xml create mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java create mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java create mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java create mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java create mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java create mode 100644 config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider create mode 100644 config/pom.xml diff --git a/client/pom.xml b/client/pom.xml index 6e43109..4da6548 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,6 +18,12 @@ 1.4-SNAPSHOT + + com.jd.platfrom.jlog + config-zk + 1.4-SNAPSHOT + + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index f6c4c06..e433d37 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -1,14 +1,21 @@ package com.jd.platform.jlog.client; -import com.jd.platform.jlog.client.task.Monitor; + +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.mdc.Mdc; +import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.client.udp.HttpSender; import com.jd.platform.jlog.client.udp.UdpClient; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.config.ConfigCenterFactory; -import com.jd.platform.jlog.common.model.CenterConfig; -import com.jd.platform.jlog.common.model.TagConfig; -import com.jd.platform.jlog.common.model.TagHandler; +import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.tag.TagHandler; +import com.jd.platform.jlog.common.utils.FastJsonUtils; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import com.jd.platform.jlog.zk.ZkConfigurator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * TracerClientStarter @@ -17,15 +24,19 @@ import com.jd.platform.jlog.common.model.TagHandler; * @date 2021-08-13 */ public class TracerClientStarter { + + private final static Logger LOGGER = LoggerFactory.getLogger(ZkConfigurator.class); + + /** * 机房 */ private Mdc mdc; - - private CenterConfig centerConfig; - - + /** + * 如果直接配置在app.properties/yaml等主配置文件 + * 可以用ConfigurationProperties直接获取有值对象 + */ private TagConfig tagConfig; @@ -42,7 +53,6 @@ public class TracerClientStarter { public static class Builder { private String appName; private Mdc mdc; - private CenterConfig centerConfig; private TagConfig tagConfig; public Builder() { @@ -58,11 +68,6 @@ public class TracerClientStarter { return this; } - public Builder setCenterConfig(CenterConfig centerConfig) { - this.centerConfig = centerConfig; - return this; - } - public Builder setTagConfig(TagConfig tagConfig) { this.tagConfig = tagConfig; return this; @@ -70,7 +75,6 @@ public class TracerClientStarter { public TracerClientStarter build() { TracerClientStarter tracerClientStarter = new TracerClientStarter(appName); - tracerClientStarter.centerConfig = centerConfig; tracerClientStarter.tagConfig = tagConfig; tracerClientStarter.mdc = mdc; return tracerClientStarter; @@ -78,14 +82,13 @@ public class TracerClientStarter { } /** - * 启动监听etcd + * 启动监听 */ - public void startPipeline() throws Exception { - //设置ConfigCenter - ConfigCenterFactory.buildConfigCenter(centerConfig); - System.out.println("tagconfig"+tagConfig.getDelimiter()); - TagHandler.buildTag(tagConfig); + public void startPipeline() { + // 校验和设置 + checkAndSetTagConfig(); + TagHandler.build(tagConfig); Context.MDC = mdc; Monitor starter = new Monitor(); @@ -100,4 +103,28 @@ public class TracerClientStarter { //开启大对象http发送 HttpSender.uploadToWorker(); } + + + /** + * 如果未赋值，从配置器获取，底层是Properties + */ + private void checkAndSetTagConfig(){ + if(tagConfig != null){ + LOGGER.info("从主配置获取的tagConfig", tagConfig.toString()); + return; + } + Configurator configurator = ConfiguratorFactory.getInstance(); + String reqTag = configurator.getConfig("reqTags"); + String logTag = configurator.getConfig("logTags"); + String regex = configurator.getConfig("regex"); + String delimiter = configurator.getConfig("delimiter"); + String join = configurator.getConfig("join"); + tagConfig = TagConfig.Builder.aTagConfig().reqTags(FastJsonUtils.toList(reqTag, String.class)) + .logTags(FastJsonUtils.toList(logTag, String.class)) + .regex(regex).delimiter(delimiter).join(join).build(); + LOGGER.info("从配置器获取的tagConfig:{}", tagConfig.toString()); + configurator.addConfigListener("jLog.properties"); + configurator.removeConfigListener("jLog.properties"); + + } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index b8bd03f..0da8bf6 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -5,6 +5,7 @@ import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.common.tag.TagHandler; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; @@ -150,16 +151,12 @@ public class HttpFilter implements Filter { long tracerId, String uri) { //request的各个入参 Map params = servletRequest.getParameterMap(); - //将request信息保存 - Map requestMap = new HashMap<>(params.size()); - for (String key : params.keySet()) { - requestMap.put(key, params.get(key)[0]); - } - requestMap.put("appName", Context.APP_NAME); - requestMap.put("serverIp", IpUtils.getIp()); - requestMap.put("tracerId", tracerId); - requestMap.put("uri", uri); - tracerObject.add(requestMap); + Map extMap = new HashMap<>(params.size()); + extMap.put("appName", Context.APP_NAME); + extMap.put("serverIp", IpUtils.getIp()); + extMap.put("tracerId", tracerId); + extMap.put("uri", uri); + tracerObject.add(TagHandler.extractReqTag(params, extMap)); } @Override diff --git a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java index ea3f62a..df03264 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java +++ b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java @@ -1,20 +1,18 @@ package com.jd.platform.jlog.client.task; -import cn.hutool.core.collection.CollectionUtil; -import com.alibaba.nacos.api.exception.NacosException; + import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.worker.WorkerInfoHolder; -import com.jd.platform.jlog.common.config.ConfigCenterEnum; -import com.jd.platform.jlog.common.config.ConfigCenterFactory; -import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.constant.Constant; -import io.grpc.StatusRuntimeException; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -27,10 +25,8 @@ import java.util.concurrent.TimeUnit; * @createTime 2022年02月12日 10:20:00 */ public class Monitor { - /** - * logger - */ - private Logger logger = LoggerFactory.getLogger(getClass()); + + private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class); /** * 开始获取workerIp地址并保存 @@ -47,8 +43,12 @@ public class Monitor { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 scheduledExecutorService.scheduleAtFixedRate(() -> { - logger.info("trying to connect to etcd and fetch worker info"); - fetch(); + LOGGER.info("trying to connect to etcd and fetch worker info"); + try { + fetch(); + } catch (Exception e) { + e.printStackTrace(); + } }, 0, 30, TimeUnit.SECONDS); } @@ -56,44 +56,32 @@ public class Monitor { /** * 从配置中心获取worker的ip集合 */ - private void fetch() { - IConfigCenter configCenter = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD); + private void fetch() throws Exception { + Configurator configurator = ConfiguratorFactory.getInstance(); //获取所有worker的ip - List keys = null; + Map keyVal = null; try { //如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有 if (Context.MDC != null) { String mdc = parseMdc(Context.MDC); - keys = configCenter.getPrefixKey(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); + keyVal = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); } - if (CollectionUtil.isEmpty(keys)) { - keys = configCenter.getPrefixKey(Constant.WORKER_PATH + Context.APP_NAME); + if (keyVal == null || keyVal.size() == 0) { + keyVal = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME); } //全是空，给个警告 - if (CollectionUtil.isEmpty(keys)) { - logger.warn("very important warn !!! workers ip info is null!!!"); + if (keyVal == null || keyVal.size() == 0) { + LOGGER.warn("very important warn !!! workers ip info is null!!!"); + return; } List addresses = new ArrayList<>(); - if (keys != null) { - for (String key : keys) { - //value里放的是ip地址 - String ipPort = null; - try { - ipPort = configCenter.get(key); - } catch (NacosException e) { - e.printStackTrace(); - } - addresses.add(ipPort); - } - } - + keyVal.forEach( (k ,v)-> addresses.add(v)); //将对应的worker保存下来 WorkerInfoHolder.mergeAndConnectNew(addresses); - } catch (StatusRuntimeException ex) { - //etcd连不上 - logger.error("etcd connected fail. Check the etcd address!!!"); + } catch (Exception ex) { + LOGGER.error("config server connected fail. Check the config address!!!"); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java index c032766..c563181 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java @@ -1,12 +1,11 @@ package com.jd.platform.jlog.client.udp; -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.collect.Queues; import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.model.TracerData; import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.AsyncWorker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,8 +112,8 @@ public class UdpSender { try { //要么key达到500个，要么达到1秒，就汇总上报给worker一次 List tempLogs = new ArrayList<>(); - Queues.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempLogs)) { + AsyncWorker.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS); + if (tempLogs.size() == 0) { continue; } diff --git a/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java b/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java index 73dff8a..e10f699 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java +++ b/client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.client.worker; -import cn.hutool.core.collection.CollectionUtil; import java.util.Collections; import java.util.List; @@ -57,7 +56,7 @@ public class WorkerInfoHolder { * address例子：10.12.139.152:11111 */ public static void mergeAndConnectNew(List allAddresses) { - if (CollectionUtil.isEmpty(allAddresses)) { + if (allAddresses.size() == 0) { WORKER_HOLDER.clear(); return; } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java index e73ba6d..8da0ed4 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.clientdemo; -import org.apache.catalina.filters.RequestFilter; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -20,6 +19,5 @@ public class ClientDemoApplication { }catch (Exception e){ e.printStackTrace(); } - } } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java similarity index 95% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java rename to clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java index 87a9b6b..234466f 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/TracerLogbackAppender.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.clientdemo.config; +package com.jd.platform.jlog.clientdemo.appender; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -9,7 +9,7 @@ import com.jd.platform.jlog.client.log.LogExceptionStackTrace; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.model.RunLogMessage; -import com.jd.platform.jlog.common.model.TagHandler; +import com.jd.platform.jlog.common.tag.TagHandler; import org.slf4j.helpers.MessageFormatter; import java.util.Map; @@ -61,10 +61,10 @@ public class TracerLogbackAppender extends AppenderBase { String formattedMessage = getMessage(loggingEvent); logMessage.setContent(formattedMessage); - Map map = TagHandler.extract(formattedMessage); + /* Map map = TagHandler.extractLogTag(formattedMessage); if(map != null){ System.out.println("Logback tag map =>> "+ JSON.toJSONString(map)); - } + }*/ return logMessage; } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java index a6a8549..d83830b 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java @@ -2,8 +2,7 @@ package com.jd.platform.jlog.clientdemo.config; import com.jd.platform.jlog.client.TracerClientStarter; import com.jd.platform.jlog.client.filter.HttpFilter; -import com.jd.platform.jlog.common.model.CenterConfig; -import com.jd.platform.jlog.common.model.TagConfig; +import com.jd.platform.jlog.common.tag.TagConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -25,17 +24,8 @@ public class DemoConfig { private Logger logger = LoggerFactory.getLogger(getClass()); - private CenterConfig centerConfig; private TagConfig tagConfig ; - public CenterConfig getCenterConfig() { - return centerConfig; - } - - public void setCenterConfig(CenterConfig centerConfig) { - this.centerConfig = centerConfig; - } - public TagConfig getTagConfig() { return tagConfig; } @@ -49,11 +39,9 @@ public class DemoConfig { TracerClientStarter tracerClientStarter = new TracerClientStarter.Builder() .setAppName("demo") - .setCenterConfig(centerConfig) .setTagConfig(tagConfig) .build(); - logger.info("init centerConfig",centerConfig); - logger.info("init tagConfig",tagConfig); + logger.info("初始化tagConfig: {}",tagConfig); tracerClientStarter.startPipeline(); } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 3a4a023..2fbd065 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -1,12 +1,14 @@ package com.jd.platform.jlog.clientdemo.web; -import com.alibaba.nacos.api.exception.NacosException; -import com.jd.platform.jlog.common.config.ConfigCenterEnum; -import com.jd.platform.jlog.common.config.ConfigCenterFactory; -import com.jd.platform.jlog.common.config.IConfigCenter; + import com.jd.platform.jlog.common.model.TracerBean; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -26,18 +28,18 @@ public class TestController { private static Logger RequestLog = LoggerFactory.getLogger("RequestLog"); @RequestMapping("/index") - public Object index() throws NacosException { + public Object index() { TracerBean tracerBean = new TracerBean(); tracerBean.setTracerId("11111"); - IConfigCenter client = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD); + Configurator configurator = ConfiguratorFactory.getInstance(); try{ - client.put("/test","val1"); + configurator.putConfig("/test","val1"); }catch (Exception e){ e.printStackTrace(); } - String val = ConfigCenterFactory.getClient(ConfigCenterEnum.ETCD).get("/test"); + String val = configurator.getConfig("/test"); System.out.println("val ===> "+val); RequestLog.info("哈哈哈哈哈哈"); @@ -45,9 +47,19 @@ public class TestController { } @RequestMapping("/log") - public Object log() throws NacosException { + public Object log() { RequestLog.info("|tag3=val3||tag4=val4||这是随便的log|"); return 1; } + + @PostMapping(value = "/test", consumes = MediaType.APPLICATION_JSON_VALUE) + public Object test(@RequestBody TestReq req) { + + + return 1; + } + + + } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java new file mode 100644 index 0000000..917c1b3 --- /dev/null +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.clientdemo.web; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName TestReq.java + * @Description TODO + * @createTime 2022年02月17日 16:40:00 + */ +public class TestReq { + + private Integer id; + + private String name; +} diff --git a/clientdemo/src/main/resources/application.properties b/clientdemo/src/main/resources/application.properties new file mode 100644 index 0000000..8d51d0c --- /dev/null +++ b/clientdemo/src/main/resources/application.properties @@ -0,0 +1 @@ +server.port=8082 \ No newline at end of file diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml index 520bd91..d4ffa15 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/clientdemo/src/main/resources/application.yml @@ -1,22 +1,11 @@ -#etcd的地址，如有多个用逗号分隔 +serverAddr: 101.42.242.201:2181 jlog: center-config: center: etcd etcdServer: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ nacosServer: 101.42.242.201:8848 zkServer: 101.42.242.201:2181 - tag-config: - reqTags: - - tag1 - - tag2 - logTags: - - tag3 - - tag4 - regex: "" - delimiter: "|" - join: = - server: - port: 8081 \ No newline at end of file + port: 8081 diff --git a/clientdemo/src/main/resources/application.ymlbak b/clientdemo/src/main/resources/application.ymlbak new file mode 100644 index 0000000..165f59c --- /dev/null +++ b/clientdemo/src/main/resources/application.ymlbak @@ -0,0 +1,22 @@ +serverAddr: 101.42.242.201:2181 +jlog: + center-config: + center: etcd + etcdServer: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ + nacosServer: 101.42.242.201:8848 + zkServer: 101.42.242.201:2181 + tag-config: + reqTags: + - tag1 + - tag2 + logTags: + - tag3 + - tag4 + regex: "" + delimiter: | + join: = + + + +server: + port: 8081 \ No newline at end of file diff --git a/clientdemo/src/main/resources/logback-spring.xml b/clientdemo/src/main/resources/logback-spring.xml index 22536e3..8b56e11 100644 --- a/clientdemo/src/main/resources/logback-spring.xml +++ b/clientdemo/src/main/resources/logback-spring.xml @@ -16,7 +16,7 @@ - + diff --git a/common/pom.xml b/common/pom.xml index aa3005d..1999299 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -17,47 +17,21 @@ 1.1.8.4 1.5.0-4 1.2.70 + 1.7.2 + + - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - org.apache.curator - curator-framework - 4.2.0 - - - org.apache.zookeeper - zookeeper - - - org.slf4j slf4j-log4j12 1.7.32 + io.protostuff protostuff-core @@ -69,57 +43,30 @@ ${protostuff.version} - - cn.hutool - hutool-all - ${hutool.version} - - - com.github.luben - zstd-jni - ${zstd.version} - org.xerial.snappy snappy-java ${snappy.version} + - io.netty - netty-all - ${netty4.version} + com.github.luben + zstd-jni + ${zstd.version} + com.alibaba fastjson ${fastjson.version} - - com.ibm.etcd - etcd-java - ${hp-etcd.version} - - - com.github.ben-manes.caffeine - guava - ${caffeine.version} - - - - com.alibaba.nacos - nacos-client - - 1.4.3 - - - com.ctrip.framework.apollo - apollo-client - 1.9.2 + io.netty + netty-all + ${netty4.version} - \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java deleted file mode 100644 index 86f9818..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/apollo/ApolloClient.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.jd.platform.jlog.common.config.apollo; - -import com.ctrip.framework.apollo.Config; -import com.ctrip.framework.apollo.ConfigService; -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.model.CenterConfig; - -import java.util.List; -import java.util.Map; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName ApolloClient.java - * @Description TODO - * @createTime 2022年02月10日 20:37:00 - */ - - -public class ApolloClient implements IConfigCenter { - - private Config config; - - public ApolloClient() {} - - public ApolloClient(Config config) { - this.config = config; - } - - @Override - public IConfigCenter buildClient(CenterConfig centerConfig){ - config = ConfigService.getAppConfig(); - return new ApolloClient(config); - } - - @Override - public void put(String key, String value) { - - } - - @Override - public String get(String key) { - return null; - } - - @Override - public void delete(String key) { - - } - - @Override - public List getPrefixKey(String key) { - return null; - } - - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java deleted file mode 100644 index 4bed987..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZKClient.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.jd.platform.jlog.common.config.zookeeper; - - -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.model.CenterConfig; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; - -import java.util.List; - -/** - * @author shenkaiwen5 - * @version 1.0 - * @date 2022-01-06 - */ -public class ZKClient implements IConfigCenter { - - private CuratorFramework curator; - - public ZKClient() {} - - public ZKClient(CuratorFramework curatorFramework) { - this.curator = curatorFramework; - } - - - @Override - public IConfigCenter buildClient(CenterConfig config) { - String zkServer = config.getZkServer(); - CuratorFramework client = CuratorFrameworkFactory.builder().connectString(zkServer) - // 连接超时时间 - .sessionTimeoutMs(10000) - // 会话超时时间 - .connectionTimeoutMs(10000) - // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build(); - client.start(); - return new ZKClient(client); - } - - @Override - public void put(String key, String value) { - try { - curator.create().creatingParentsIfNeeded() // 若创建节点的父节点不存在则先创建父节点再创建子节点 - .withMode(CreateMode.EPHEMERAL) // 创建的是临时节点 - .withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE) // 默认匿名权限,权限scheme id:'world,'anyone,:cdrwa - .forPath(key, value.getBytes()); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - - @Override - public void delete(String key) { } - - @Override - public String get(String key) { - try { - byte[] bt = curator.getData().forPath(key); - return new String(bt, "utf-8"); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - - @Override - public List getPrefixKey(String key) { - try { - return curator.getChildren().forPath(key); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java b/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java deleted file mode 100644 index 95d453a..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/zookeeper/ZookeeperTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.jd.platform.jlog.common.config.zookeeper; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; - -public class ZookeeperTest { - public static void main(String[] args) throws Exception { - ZooKeeper zk = new ZooKeeper("101.42.242.201:2181", 3000, null); - - System.out.println(zk.getClass()); - if(zk.exists("/test22", false) == null) - { - zk.create("/test22", "znode1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - System.out.println(new String(zk.getData("/test22", false, null))); - } - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index cc6e197..9429a1b 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -1,8 +1,5 @@ package com.jd.platform.jlog.common.constant; -import com.alibaba.fastjson.JSON; -import com.alibaba.nacos.common.utils.StringUtils; - import java.util.HashSet; import java.util.Set; diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java b/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java deleted file mode 100644 index 7280215..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/model/CenterConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.jd.platform.jlog.common.model; - -import com.alibaba.nacos.common.utils.StringUtils; -import com.jd.platform.jlog.common.config.ConfigCenterEnum; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName JConfig.java - * @Description TODO - * @createTime 2022年02月11日 23:11:00 - */ -public class CenterConfig { - - private String etcdServer; - - private String nacosServer; - - private String zkServer; - - - public String getEtcdServer() { - return etcdServer; - } - - public void setEtcdServer(String etcdServer) { - this.etcdServer = etcdServer; - } - - public String getNacosServer() { - return nacosServer; - } - - public void setNacosServer(String nacosServer) { - this.nacosServer = nacosServer; - } - - public String getZkServer() { - return zkServer; - } - - public void setZkServer(String zkServer) { - this.zkServer = zkServer; - } - - - @Override - public String toString() { - return "CenterConfig{" + - ", etcdServer='" + etcdServer + '\'' + - ", nacosServer='" + nacosServer + '\'' + - ", zkServer='" + zkServer + '\'' + - '}'; - } -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java similarity index 97% rename from common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java rename to common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java index 7887eca..5b92ccf 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TagConfig.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.common.model; +package com.jd.platform.jlog.common.tag; import java.io.Serializable; import java.util.List; @@ -7,7 +7,6 @@ import java.util.List; * @author tangbohu * @version 1.0.0 * @ClassName TagConfig.java - * @Description TODO * @createTime 2022年02月13日 22:35:00 */ public class TagConfig implements Serializable { diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java similarity index 50% rename from common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java rename to common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java index 5d333b3..cf4c819 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java @@ -1,7 +1,10 @@ -package com.jd.platform.jlog.common.model; +package com.jd.platform.jlog.common.tag; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.common.utils.ConfigUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.sun.istack.internal.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,19 +20,16 @@ import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; * @author tangbohu * @version 1.0.0 * @ClassName Tag.java - * @Description TODO * @createTime 2022年02月12日 21:28:00 */ public class TagHandler { - private static Logger logger = LoggerFactory.getLogger(TagHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TagHandler.class); private Set reqTags; private Set logTags; - private String regex; - private String delimiter = "|"; private int delimiterLen = delimiter.length(); @@ -38,61 +38,85 @@ public class TagHandler { private Pattern pattern; - private static TagHandler INSTANCE = null; + private static volatile TagHandler INSTANCE = null; + + + /** + * 构建标签处理器 + * @param tagConfig 配置类 + */ + public static void build(TagConfig tagConfig) { - public static void buildTag(TagConfig tagConfig) { + TagHandler handler = new TagHandler(); String regex = tagConfig.getRegex(); - TagHandler handler = new TagHandler(); handler.reqTags = new HashSet<>(tagConfig.getReqTags()); handler.logTags = new HashSet<>(tagConfig.getLogTags()); - if(regex != null && !"".equals(regex)){ - handler.regex = regex; + if(StringUtil.isNotEmpty(regex)){ handler.pattern = Pattern.compile(regex); }else{ - handler.delimiter = tagConfig.getDelimiter(); - handler.delimiterLen = tagConfig.getDelimiter().length(); - handler.join = tagConfig.getJoin(); - System.out.println("delimiter==> "+tagConfig.getDelimiter()); - String escapeDelimiter = ConfigUtil.escapeExprSpecialWord(tagConfig.getDelimiter()); - System.out.println("## escapeDelimiter==> "+escapeDelimiter); - - // String str = "("+ formatDelimiter +"[\\w\\W]*?"+ formatDelimiter +")"; String str = String.format("(%s[\\w\\W]*?%s)", escapeDelimiter, escapeDelimiter); - System.out.println("这里 实力话了"+str); - handler.pattern = Pattern.compile(str); } + handler.delimiter = tagConfig.getDelimiter(); + handler.delimiterLen = tagConfig.getDelimiter().length(); + handler.join = tagConfig.getJoin(); INSTANCE = handler; + //LOGGER.info("构建标签处理器单例完成:{}",INSTANCE.toString()); } + /** + * 提取请求参数里的标签 + * @param params 参数 + * @param ext 额外附加的,如ip等 + * @return tags + */ + public static Map extractReqTag(Map params, @NotNull Map ext) { + + if(INSTANCE == null){ return null; } + Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); + for (String tag : INSTANCE.reqTags) { + Object val = ext.get(tag); + if(val != null){ + requestMap.put(tag, val); + continue; + } - public static Map extract(String content) { - if(INSTANCE == null){ + if(CollectionUtil.isNotEmpty(params) && params.get(tag) != null){ + requestMap.put(tag, params.get(tag)[0]); + } + } + + return requestMap; + } + + + /** + * 提取普通日志中的标签 + * @param content 内容 + * @return tags + */ + public static Map extractLogTag(String content) { + if(INSTANCE == null || content.length() < 1){ return null; } Map tagMap = new HashMap<>(3); Matcher m = INSTANCE.pattern.matcher(content); - System.out.println("content==> "+content); - while (m.find()) { String str = m.group().substring(INSTANCE.delimiterLen, m.group().length() - INSTANCE.delimiterLen); if(str.contains(INSTANCE.join)){ String[] arr = str.split(INSTANCE.join); - System.out.println("INSTANCE.logTags==> "+INSTANCE.logTags); if(INSTANCE.logTags.contains(arr[0])){ tagMap.put(arr[0], arr[1]); } }else if(str.length() < TAG_NORMAL_KEY_MAX_LEN){ - System.out.println("INSTANCE.TAG_NORMAL_KEY==> "+str); - tagMap.put(TAG_NORMAL_KEY, str); }else{ if (RANDOM.nextInt(50) == 1) { - logger.info("Some logs lack tags and are larger than 20 in length. Therefore, they are simply stored"); + LOGGER.info("Some logs lack tags and are larger than 20 in length. Therefore, they are simply stored"); } } } @@ -101,9 +125,28 @@ public class TagHandler { + /** + * 刷新标签处理器 + * @param tagConfig 新的配置 + */ + public static void refresh(TagConfig tagConfig) { + INSTANCE = null; + build(tagConfig); + } + - // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); + @Override + public String toString() { + return "TagHandler{" + + "reqTags=" + reqTags + + ", logTags=" + logTags + + ", delimiter='" + delimiter + '\'' + + ", delimiterLen=" + delimiterLen + + ", join='" + join + '\'' + + '}'; + } + // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); public static List extractTest(String content) { List list = new ArrayList<>(); @@ -111,7 +154,7 @@ public class TagHandler { while (m.find()) { list.add(m.group().substring(1, m.group().length() - 1)); // list.add(m.group().substring(2, m.group().length() - 2)); - // list.add(m.group().substring(3, m.group().length()-3)); + } return list; } @@ -121,58 +164,16 @@ public class TagHandler { // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|\\|[\\w\\W]*?\\|\\|)"); static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); - static String str1 = "|订单完成|订单类型=1"; - static String str2 = " ddddd"; - - // String reg=".*ll.*"; //判断字符串中是否含有ll - // System.out.println(str.matches(reg)); - - /* static String str1 = "||a=1||b=2||qwewe"; + static String str1 = "||a=1||b=2||qwewe"; static String str2 = "||a=1||b=2||qwewe||"; static String str3 = "||a=1||eee||b=2"; - static String str4 = "||a=1||eee||b=2||";*/ + static String str4 = "||a=1||eee||b=2||"; - // static String str3 = "|a=1|这是普通log|b=2"; - // static String str4 = "|a=1|这是普通log|b=2|"; - public static void main(String[] args) { - // System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1))); + System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1))); System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2))); - // System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3))); - // System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4))); + System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3))); + System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4))); } - - - - - /* Map requestMap = new HashMap<>(16); - requestMap.put("tracerId", "1"); - requestMap.put("pin", UUID.randomUUID()); - requestMap.put("appName", "myTest"); - requestMap.put("uuid", "uuid" + UUID.randomUUID()); - requestMap.put("client", "android"); - requestMap.put("clientVersion", "10.3.2"); - requestMap.put("ip", "127.0.0.1"); - requestMap.put("serverIp", "127.0.0.1"); - requestMap.put("uri", "test"); - tracerObject.add(requestMap); - - - - /* @Bean - public IConfigCenter client() throws Exception { - CenterConfig jConfig = new CenterConfig.Builder() - .etcdServer(etcdServer) - .nacosServer(nacosServer) - .zkServer(zkServer) - .build(); - ConfigCenterFactory.buildConfigCenter(jConfig); - logger.info("jConfig : " + jConfig); - //连接多个时，逗号分隔 - //return JdEtcdBuilder.build(etcdServer); - return ConfigCenterFactory.getClient(jConfig.getCenter()); - }*/ - - } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java index bae815c..11313ec 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java @@ -1,7 +1,10 @@ package com.jd.platform.jlog.common.utils; +import java.util.Collection; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * 异步线程池AsyncPool @@ -19,4 +22,6 @@ public class AsyncPool { public static void shutDown() { threadPoolExecutor.shutdown(); } + + } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java new file mode 100644 index 0000000..5f340fd --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java @@ -0,0 +1,37 @@ +package com.jd.platform.jlog.common.utils; + +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName AsyncWorker.java + * @Description TODO + * @createTime 2022年02月21日 17:10:00 + */ +public class AsyncWorker { + + public static int drain(BlockingQueue q, Collection buffer, int numElements, long timeout, TimeUnit unit) throws Exception { + if(buffer == null){ + throw new Exception("[Assertion failed] - the buffer argument cannot be null"); + } + long deadline = System.nanoTime() + unit.toNanos(timeout); + int added = 0; + + while(added < numElements) { + added += q.drainTo(buffer, numElements - added); + if (added < numElements) { + E e = q.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS); + if (e == null) { + break; + } + buffer.add(e); + ++added; + } + } + return added; + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java new file mode 100644 index 0000000..426b219 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java @@ -0,0 +1,84 @@ +package com.jd.platform.jlog.common.utils; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CollectionUtil.java + * @Description TODO + * @createTime 2022年02月21日 17:14:00 + */ +public class CollectionUtil { + + public static boolean isEmpty(Collection collection) { + return collection == null || collection.isEmpty(); + } + + public static boolean isNotEmpty(Collection collection) { + return collection != null && collection.isEmpty(); + } + + + public static boolean isEmpty(Map map) { + return !isNotEmpty(map); + } + + + public static boolean isNotEmpty(Map map) { + return map != null && !map.isEmpty(); + } + + + public static boolean equals(List c1, List c2) { + if (c1 == null && c2 == null) { + return true; + } else if (c1 != null && c2 != null) { + if (c1.size() != c2.size()) { + return false; + } else { + for(int i = 0; i < c1.size(); ++i) { + if (!Objects.equals(c1.get(i), c2.get(i))) { + return false; + } + } + + return true; + } + } else { + return false; + } + } + + public static boolean equals(Map c1, Map c2) { + if (c1 == null && c2 == null) { + return true; + } else if (c1 != null && c2 != null) { + if (c1.size() != c2.size()) { + return false; + } else { + Iterator var2 = c1.entrySet().iterator(); + + Object v1; + Object v2; + do { + if (!var2.hasNext()) { + return true; + } + + Map.Entry entry = (Map.Entry)var2.next(); + K k = entry.getKey(); + v1 = entry.getValue(); + v2 = c2.get(k); + } while(Objects.equals(v1, v2)); + + return false; + } + } else { + return false; + } + } + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java index e42a753..e7e1d67 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -1,13 +1,6 @@ package com.jd.platform.jlog.common.utils; -import com.alibaba.nacos.common.utils.StringUtils; -import com.jd.platform.jlog.common.config.ConfigCenterEnum; import com.jd.platform.jlog.common.constant.Constant; -import com.jd.platform.jlog.common.model.CenterConfig; - -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.concurrent.ThreadLocalRandom; /** @@ -38,6 +31,7 @@ public class ConfigUtil { return str; } +/* public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception { Class clz = config.getClass(); @@ -57,6 +51,7 @@ public class ConfigUtil { } throw new Exception("Configuration center cannot be found"); } +*/ private static String getMethodName(String fieldName) { diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java b/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java index 9cb37dc..4b60305 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java @@ -1,6 +1,6 @@ package com.jd.platform.jlog.common.utils; -import com.google.common.base.Preconditions; +import com.sun.tools.javac.util.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,14 +86,14 @@ public class IdWorker { * @param workerId 工作进程Id */ private static void setWorkerId(final Long workerId) { - Preconditions.checkArgument(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE); + Assert.check(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE); IdWorker.workerId = workerId; } //下一个ID生成算法 public static long nextId() { long time = System.currentTimeMillis(); - Preconditions.checkState(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds", lastTime, time); + Assert.check(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds"+lastTime); if (lastTime == time) { if (0L == (sequence = ++sequence & SEQUENCE_MASK)) { time = waitUntilNextTime(time); diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java new file mode 100644 index 0000000..09aaa5f --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java @@ -0,0 +1,45 @@ +package com.jd.platform.jlog.common.utils; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName StringUtil.java + * @Description TODO + * @createTime 2022年02月21日 17:13:00 + */ +public class StringUtil { + + + public StringUtil() { + } + + public static boolean isEmpty(CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static boolean isNotEmpty(CharSequence cs) { + return !isEmpty(cs); + } + + + public static boolean isBlank(CharSequence cs) { + int strLen; + if (cs != null && (strLen = cs.length()) != 0) { + for(int i = 0; i < strLen; ++i) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; + } + } + + return true; + } else { + return true; + } + } + + public static boolean isNotBlank(CharSequence cs) { + return !isBlank(cs); + } + + +} diff --git a/config/config-apollo/pom.xml b/config/config-apollo/pom.xml new file mode 100644 index 0000000..5156749 --- /dev/null +++ b/config/config-apollo/pom.xml @@ -0,0 +1,42 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-apollo + config-apollo + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + slf4j-api + org.slf4j + + + + + com.ctrip.framework.apollo + apollo-client + 1.9.2 + + + slf4j-api + org.slf4j + + + guava + com.google.guava + + + + + + diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java new file mode 100644 index 0000000..df9361a --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -0,0 +1,172 @@ +package com.jd.platform.jlog.config.apollo; + +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.*; + +import com.ctrip.framework.apollo.core.utils.StringUtils; +import com.ctrip.framework.apollo.enums.PropertyChangeType; +import com.ctrip.framework.apollo.model.ConfigChange; +import com.ctrip.framework.apollo.spring.property.AutoUpdateConfigChangeListener; +import com.jd.platform.jlog.core.*; +import com.ctrip.framework.apollo.Config; +import com.ctrip.framework.apollo.ConfigService; + +import static com.ctrip.framework.apollo.core.ApolloClientSystemConsts.APP_ID; +import static com.jd.platform.jlog.config.apollo.ApolloConstant.*; + + +/** + * @author didi + */ +public class ApolloConfigurator implements Configurator { + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + private static volatile Config config; + private static final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(); + private static volatile ApolloConfigurator instance; + + private ApolloConfigurator() { + loadApolloServerConfig(); + if (config == null) { + synchronized (ApolloConfigurator.class) { + if (config == null) { + config = ConfigService.getConfig(FILE_CONFIG.getConfig(DEFAULT_NAMESPACE)); + config.addChangeListener(changeEvent -> { + for (String key : changeEvent.changedKeys()) { + if (!CONFIG_LISTENER_MAP.containsKey(key)) { + continue; + } + ConfigChange change = changeEvent.getChange(key); + ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), + change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType())); + CONFIG_LISTENER_MAP.get(key).onProcessEvent(event); + } + }); + } + } + } + } + + /** + * Gets instance. + * + * @return the instance + */ + public static ApolloConfigurator getInstance() { + if (instance == null) { + synchronized (ApolloConfigurator.class) { + if (instance == null) { + instance = new ApolloConfigurator(); + } + } + } + return instance; + } + + @Override + public String getConfig(String key) { + return null; + } + + @Override + public String getConfig(String key, long timeoutMills) { + String value = System.getProperty(key); + if (value != null) { + return value; + } + + return config.getProperty(key,""); + } + + @Override + public boolean putConfig(String key, String content) { + return false; + } + + + @Override + public boolean putConfig(String dataId, String content, long timeoutMills) { + return false; + } + + + + @Override + public boolean removeConfig(String dataId, long timeoutMills) { + return false; + } + + + @Override + public boolean removeConfig(String key) { + return false; + } + + @Override + public void addConfigListener(String key) { + if (StringUtils.isBlank(key)) { + return; + } + CONFIG_LISTENER_MAP.put(key, new ApolloListener()); + } + + @Override + public void removeConfigListener(String key) { + CONFIG_LISTENER_MAP.remove(key); + } + + + @Override + public ConfigChangeListener getConfigListeners(String dataId) { + return CONFIG_LISTENER_MAP.get(dataId); + } + + + @Override + public String getType() { + return "apollo"; + } + + + @Override + public Map getConfigByPrefix(String prefix) { + return null; + } + + + private ConfigChangeType getChangeType(PropertyChangeType changeType) { + switch (changeType) { + case ADDED: + return ConfigChangeType.ADD; + case DELETED: + return ConfigChangeType.DELETE; + default: + return ConfigChangeType.MODIFY; + } + } + + + private void loadApolloServerConfig() { + Properties properties = System.getProperties(); + if (!properties.containsKey(PROP_APP_ID)) { + System.setProperty(PROP_APP_ID, FILE_CONFIG.getConfig(APP_ID)); + } + if (!properties.containsKey(PROP_APOLLO_META)) { + System.setProperty(PROP_APOLLO_META, FILE_CONFIG.getConfig(APOLLO_META)); + } + if (!properties.containsKey(PROP_APOLLO_SECRET)) { + String secretKey = FILE_CONFIG.getConfig(APOLLO_SECRET); + if (!StringUtils.isBlank(secretKey)) { + System.setProperty(PROP_APOLLO_SECRET, secretKey); + } + } + if (!properties.containsKey(APOLLO_CLUSTER)) { + System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getConfig(APOLLO_CLUSTER)); + } + if (!properties.containsKey(APOLLO_CONFIG_SERVICE)) { + System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getConfig(APOLLO_CONFIG_SERVICE)); + } + } +} diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java new file mode 100644 index 0000000..9288be1 --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.config.apollo; + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +/** + * @author didi + */ +public class ApolloConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return ApolloConfigurator.getInstance(); + } +} + diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java new file mode 100644 index 0000000..39833b4 --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java @@ -0,0 +1,22 @@ +package com.jd.platform.jlog.config.apollo; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloConstant.java + * @Description TODO + * @createTime 2022年02月22日 11:06:00 + */ +public class ApolloConstant { + + static final String APOLLO_META = "apolloMeta"; + static final String APOLLO_SECRET = "apolloAccesskeySecret"; + static final String APOLLO_CLUSTER = "jLog"; + static final String APOLLO_CONFIG_SERVICE = "apolloConfigService"; + static final String PROP_APP_ID = "app.id"; + static final String PROP_APOLLO_META = "apollo.meta"; + static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.configService"; + static final String PROP_APOLLO_SECRET = "apollo.accesskey.secret"; + static final String PROP_APOLLO_CLUSTER = "apollo.cluster"; + static final String DEFAULT_NAMESPACE = "jLog"; +} diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java new file mode 100644 index 0000000..fb1b268 --- /dev/null +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java @@ -0,0 +1,21 @@ +package com.jd.platform.jlog.config.apollo; + +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloListener.java + * @Description TODO + * @createTime 2022年02月22日 19:18:00 + */ +public class ApolloListener implements ConfigChangeListener { + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + + + } +} diff --git a/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000..f46c96c --- /dev/null +++ b/config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.config.apollo.ApolloConfiguratorProvider \ No newline at end of file diff --git a/config/config-core/pom.xml b/config/config-core/pom.xml new file mode 100644 index 0000000..16231fa --- /dev/null +++ b/config/config-core/pom.xml @@ -0,0 +1,21 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-core + config-core + + + + com.jd.platfrom.jlog + common + 1.4-SNAPSHOT + + + + diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java new file mode 100644 index 0000000..c9113f3 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -0,0 +1,139 @@ +package com.jd.platform.jlog.core; + + +/** + * @author didi + */ +public class ConfigChangeEvent { + + private String key; + private String oldValue; + private String newValue; + private String namespace; + private ConfigChangeType changeType; + private static final String DEFAULT_NAMESPACE = "DEFAULT"; + + + public ConfigChangeEvent(){ + + } + + public ConfigChangeEvent(String key, String newValue) { + this(key, DEFAULT_NAMESPACE, null, newValue, ConfigChangeType.MODIFY); + } + + public ConfigChangeEvent(String key, String namespace, String oldValue, String newValue, + ConfigChangeType type) { + this.key = key; + this.namespace = namespace; + this.oldValue = oldValue; + this.newValue = newValue; + this.changeType = type; + } + + /** + * Gets data id. + * + * @return the data id + */ + public String getKey() { + return key; + } + + /** + * Sets data id. + * + * @param key key + */ + public ConfigChangeEvent setKey(String key) { + this.key = key; + return this; + } + + /** + * Gets old value. + * + * @return the old value + */ + public String getOldValue() { + return oldValue; + } + + /** + * Sets old value. + * + * @param oldValue the old value + */ + public ConfigChangeEvent setOldValue(String oldValue) { + this.oldValue = oldValue; + return this; + } + + /** + * Gets new value. + * + * @return the new value + */ + public String getNewValue() { + return newValue; + } + + /** + * Sets new value. + * + * @param newValue the new value + */ + public ConfigChangeEvent setNewValue(String newValue) { + this.newValue = newValue; + return this; + } + + /** + * Gets change type. + * + * @return the change type + */ + public ConfigChangeType getChangeType() { + return changeType; + } + + /** + * Sets change type. + * + * @param changeType the change type + */ + public ConfigChangeEvent setChangeType(ConfigChangeType changeType) { + this.changeType = changeType; + return this; + } + + /** + * Gets namespace. + * + * @return the namespace + */ + public String getNamespace() { + return namespace; + } + + /** + * Sets namespace. + * + * @param namespace the namespace + */ + public ConfigChangeEvent setNamespace(String namespace) { + this.namespace = namespace; + return this; + } + + @Override + public String toString() { + return "ConfigChangeEvent{" + + "key='" + key + '\'' + + ", oldValue='" + oldValue + '\'' + + ", newValue='" + newValue + '\'' + + ", namespace='" + namespace + '\'' + + ", changeType=" + changeType + + '}'; + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java new file mode 100644 index 0000000..4e8c81f --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -0,0 +1,65 @@ +/* + * Copyright 1999-2019 Seata.io Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jd.platform.jlog.core; + + +import java.util.concurrent.*; + + +/** + * @author didi + */ +public interface ConfigChangeListener { + + + ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(1, 1, + Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), Executors.defaultThreadFactory()); + + + void onChangeEvent(ConfigChangeEvent event); + + + default void onProcessEvent(ConfigChangeEvent event) { + getExecutorService().submit(() -> { + beforeEvent(); + try { + onChangeEvent(event); + } catch (Exception e) { + e.printStackTrace(); + } + afterEvent(); + }); + } + + default void onShutDown() { + getExecutorService().shutdownNow(); + } + + + default ExecutorService getExecutorService() { + return EXECUTOR_SERVICE; + } + + + default void beforeEvent() { + + } + + + default void afterEvent() { + + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java new file mode 100644 index 0000000..a799103 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java @@ -0,0 +1,11 @@ +package com.jd.platform.jlog.core; + + +public enum ConfigChangeType { + + ADD, + + MODIFY, + + DELETE +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java new file mode 100644 index 0000000..45cae7e --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -0,0 +1,90 @@ +package com.jd.platform.jlog.core; + +import java.util.Map; + +/** + * @author tangbohu + */ +public interface Configurator { + + /** + * 获取配置 + * @param key key + * @return val + */ + String getConfig(String key); + + /** + * 获取配置 + * @param key key + * @param timeoutMills timeoutMills + * @return val + */ + String getConfig(String key, long timeoutMills); + + /** + * 设置配置 + * @param key key + * @return content val + */ + boolean putConfig(String key, String content); + + /** + * 设置配置 + * @param key key + * @return content val + * @param timeoutMills timeoutMills + */ + boolean putConfig(String key, String content, long timeoutMills); + + + /** + * 移除配置 + * @param key key + * @return val + */ + boolean removeConfig(String key); + + /** + * 移除配置 + * @param key key + * @param timeoutMills timeoutMills + * @return val + */ + boolean removeConfig(String key, long timeoutMills); + + + /** + * 添加监听器 + * @param key key + */ + void addConfigListener(String key); + + + /** + * 移除监听器 + * @param key key + */ + void removeConfigListener(String key); + + + /** + * 获取对应监听器 + * @param key key + * @return Listeners + */ + ConfigChangeListener getConfigListeners(String key); + + /** + * 获取配置器类型 + * @return string example:apollo + */ + String getType(); + + /** + * 根据前缀/父级路径获取子节点 + * @param prefix + * @return List + */ + Map getConfigByPrefix(String prefix); +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java new file mode 100644 index 0000000..664f130 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 1999-2019 Seata.io Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jd.platform.jlog.core; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ServiceLoader; + + +/** + * @author tangbohu + */ +public class ConfiguratorFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConfiguratorFactory.class); + + + private static volatile Configurator instance = null; + + + public static volatile Configurator base = null; + + + + public static Configurator getInstance(){ + if (instance == null) { + synchronized (Configurator.class) { + if (instance == null) { + instance = buildConfiguration(); + LOGGER.info("构建总配置器单例完成 instance 获取reqTags结果:{}", instance.getConfig("reqTags")); + LOGGER.info("构建总配置器单例完成 Base 获取serverAddr结果:{}", base.getConfig("serverAddr")); + } + } + } + return instance; + } + + private static Configurator buildConfiguration() { + + try { + base = new FileConfigurator(); + } catch (IOException e) { + e.printStackTrace(); + } + + ServiceLoader builders = ServiceLoader.load(ConfiguratorProvider.class); + LOGGER.info("ServiceLoader获取到的实现类:{}", builders.toString()); + for (ConfiguratorProvider provider : builders) { + LOGGER.info("第一个实现类类型为:{}", provider.build().getType()); + return provider.build(); + } + return base; + } + + protected static void reload() throws Exception { + instance = null; + getInstance(); + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java new file mode 100644 index 0000000..82b7e59 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java @@ -0,0 +1,14 @@ +package com.jd.platform.jlog.core; + + +/** + * @author tangbohu + */ +public interface ConfiguratorProvider { + + /** + * + * @return + */ + Configurator build(); +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java new file mode 100644 index 0000000..4fa22ab --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -0,0 +1,242 @@ +package com.jd.platform.jlog.core; + +import com.jd.platform.jlog.common.utils.StringUtil; +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URL; +import java.util.*; +import java.util.concurrent.*; + + +/** + * @author didi + */ +public class FileConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class); + + private static final long LISTENER_CONFIG_INTERVAL = 5000; + + private static final String CONFIG_FILE_PROPERTIES = "/application.properties"; + + private static final String CONFIG_FILE_YML = "/application.yml"; + + private static final String JLOG_CONFIG_FILE = "/jLog.properties"; + + private static final String[] FILE_ARRAY = { CONFIG_FILE_PROPERTIES, CONFIG_FILE_YML, JLOG_CONFIG_FILE }; + + private final ConcurrentMap configListenerMap = new ConcurrentHashMap<>(8); + + private final Map listenedConfigMap = new HashMap<>(8); + + private final FileListener fileListener = new FileListener(); + + private final Properties properties = new Properties(); + + private volatile long lastModify = 0L; + + + public FileConfigurator() throws IOException { + + String env = System.getenv("env"); + + for (String file : FILE_ARRAY) { + + file = StringUtil.isEmpty(env) ? file : file + "_" + env; + + try (InputStream fis = this.getClass().getResourceAsStream(file)) { + if(fis != null){ + properties.load(fis); + LOGGER.info("{}配置文件配置:{}", file, properties.toString()); + }else{ + LOGGER.warn("{}配置文件为空",file); + } + } + } + LOGGER.info("合并后的配置:{}",properties.toString()); + } + + + @Override + public String getConfig(String key) { + return properties.getProperty(key); + } + + + @Override + public String getConfig(String key, long timeoutMills) { + String value = System.getProperty(key); + if (value != null) { + return value; + } + + if(properties.size() == 0){ + // 不正常 warn + return null; + } + return properties.getProperty(key); + } + + @Override + public boolean putConfig(String key, String content) { + return putConfig(key,content,2000L); + } + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + return false; + } + + + @Override + public boolean removeConfig(String key, long timeoutMills) { + return false; + } + + + @Override + public void addConfigListener(String key) { + if (StringUtil.isBlank(key)) { + return; + } + FileListener fileListener = new FileListener(); + configListenerMap.put(key, fileListener); + listenedConfigMap.put(key, ConfiguratorFactory.getInstance().getConfig(key,1000L)); + fileListener.addListener(key, fileListener); + } + + + @Override + public boolean removeConfig(String key) { + return false; + } + + + + @Override + public void removeConfigListener(String key) { + ConfigChangeListener configListener = getConfigListeners(key); + configListenerMap.remove(key); + listenedConfigMap.remove(key); + configListener.onShutDown(); + } + + + + @Override + public ConfigChangeListener getConfigListeners(String key) { + return configListenerMap.get(key); + } + + + + @Override + public String getType() { + return "file"; + } + + @Override + public Map getConfigByPrefix(String prefix) { + return null; + } + + + + class FileListener implements ConfigChangeListener { + + private final Map> keyListenersMap = new HashMap<>(); + + private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("fileListener", 1)); + + FileListener() {} + + public synchronized void addListener(String dataId, ConfigChangeListener listener) { + if (keyListenersMap.isEmpty()) { + fileListener.onProcessEvent(new ConfigChangeEvent()); + } + + keyListenersMap.computeIfAbsent(dataId, value -> new HashSet<>()).add(listener); + } + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + while (true) { + for (String key : keyListenersMap.keySet()) { + try { + checkAndConfigure(); + String currentConfig = properties.getProperty(key); + if (currentConfig != null) { + String oldConfig = listenedConfigMap.get(key); + if (currentConfig.equals(oldConfig)) { + listenedConfigMap.put(key, currentConfig); + event.setKey(key).setNewValue(currentConfig).setOldValue(oldConfig); + + for (ConfigChangeListener listener : keyListenersMap.get(key)) { + listener.onChangeEvent(event); + } + } + } + } catch (Exception exx) { + LOGGER.error("fileListener execute error, key :{}", key, exx); + } + } + try { + Thread.sleep(LISTENER_CONFIG_INTERVAL); + } catch (InterruptedException e) { + LOGGER.error("fileListener thread sleep error:{}", e.getMessage()); + } + } + } + + @Override + public ExecutorService getExecutorService() { + return executor; + } + } + + + + + private void checkAndConfigure() { + + File file; + FileInputStream inputStream = null; + + try { + file = new File(JLOG_CONFIG_FILE); + + long l = file.lastModified(); + + if(l <= this.lastModify){ + return; + } + this.lastModify = l; + inputStream = new FileInputStream(file); + this.properties.clear(); + this.properties.load(inputStream); + inputStream.close(); + } catch (Exception e) { + if (e instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (InterruptedIOException oe) { + Thread.currentThread().interrupt(); + } catch (Throwable ignored) { + + } + } + } + } + + + +} diff --git a/config/config-etcd/pom.xml b/config/config-etcd/pom.xml new file mode 100644 index 0000000..4c6fa9e --- /dev/null +++ b/config/config-etcd/pom.xml @@ -0,0 +1,26 @@ + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-etcd + config-etcd + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + com.ibm.etcd + etcd-java + 0.0.16 + + + + diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java new file mode 100644 index 0000000..61d198b --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java @@ -0,0 +1,12 @@ +package com.jd.platform.jlog.etcd; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +public class EtcdConfigurationProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return EtcdConfigurator.getInstance(); + } +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java new file mode 100644 index 0000000..57c5b1c --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -0,0 +1,137 @@ +package com.jd.platform.jlog.etcd; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArraySet; + +import com.google.protobuf.ByteString; +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.api.RangeResponse; +import com.ibm.etcd.client.EtcdClient; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.core.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author didi + */ +public class EtcdConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(EtcdConfigurator.class); + + private static volatile EtcdConfigurator instance; + + static volatile EtcdClient client; + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + private static final String SERVER_ADDR_KEY = "serverAddr"; + private ConcurrentMap configListenerMap = new ConcurrentHashMap<>(8); + + + private EtcdConfigurator() { + LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); + client = EtcdClient.forEndpoints(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,2000L)).withPlainText().build(); + } + + + public static EtcdConfigurator getInstance() { + if (instance == null) { + synchronized (EtcdConfigurator.class) { + if (instance == null) { + instance = new EtcdConfigurator(); + } + } + } + return instance; + } + + + @Override + public String getConfig(String key) { + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(key)).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return null; + } + return keyValues.get(0).getValue().toStringUtf8(); + } + + @Override + public String getConfig(String key, long timeoutMills) { + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(key)).timeout(timeoutMills).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return null; + } + return keyValues.get(0).getValue().toStringUtf8(); + } + + @Override + public boolean putConfig(String key, String content) { + client.getKvClient().put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(content)).sync(); + return true; + } + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + client.getKvClient().put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(content)).timeout(timeoutMills).sync(); + return true; + } + + @Override + public boolean removeConfig(String key) { + client.getKvClient().delete(ByteString.copyFromUtf8(key)).sync(); + return false; + } + + + @Override + public boolean removeConfig(String key, long timeoutMills) { + client.getKvClient().delete(ByteString.copyFromUtf8(key)).timeout(timeoutMills).sync(); + return false; + } + + @Override + public void addConfigListener(String key) { + EtcdListener etcdListener = new EtcdListener(key); + configListenerMap.put(key, etcdListener); + etcdListener.onProcessEvent(new ConfigChangeEvent()); + } + + @Override + public void removeConfigListener(String key) { + ConfigChangeListener configListener = getConfigListeners(key); + configListenerMap.remove(key); + configListener.onShutDown(); + } + + + @Override + public ConfigChangeListener getConfigListeners(String dataId) { + return configListenerMap.get(dataId); + } + + + @Override + public String getType() { + return "etcd"; + } + + @Override + public Map getConfigByPrefix(String prefix) { + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(prefix)).asPrefix().sync(); + List list = rangeResponse.getKvsList(); + Map map = new HashMap<>(list.size()); + for (KeyValue kv : list) { + map.put(kv.getKey().toStringUtf8(), kv.getValue().toStringUtf8()); + } + return map; + } + +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java new file mode 100644 index 0000000..2b9d734 --- /dev/null +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -0,0 +1,60 @@ +package com.jd.platform.jlog.etcd; + +import com.google.protobuf.ByteString; +import com.ibm.etcd.api.Event; +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.client.EtcdClient; +import com.ibm.etcd.client.kv.KvClient; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import com.jd.platform.jlog.core.ConfigChangeType; +import io.netty.util.concurrent.DefaultThreadFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName EtcdListener.java + * @Description TODO + * @createTime 2022年02月21日 23:34:00 + */ +public class EtcdListener implements ConfigChangeListener { + private final String key; + private KvClient.WatchIterator iterator; + private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("etcdListener", 1)); + + public EtcdListener(String key) { + this.key = key; + } + + + @Override + public void onShutDown() { + this.iterator.close(); + getExecutorService().shutdownNow(); + } + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(key)).start(); + while (iterator.hasNext()){ + Event eve = iterator.next().getEvents().get(0); + KeyValue kv = eve.getKv(); + Event.EventType eveType = eve.getType(); + ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; + event.setKey(key).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + this.onChangeEvent(event); + } + } + + @Override + public ExecutorService getExecutorService() { + return executor; + } +} diff --git a/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000..32851cd --- /dev/null +++ b/config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.etcd.EtcdConfigurationProvider \ No newline at end of file diff --git a/config/config-nacos/pom.xml b/config/config-nacos/pom.xml new file mode 100644 index 0000000..5878cbf --- /dev/null +++ b/config/config-nacos/pom.xml @@ -0,0 +1,27 @@ + + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-nacos + config-nacos + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + com.alibaba.nacos + nacos-client + 2.0.4 + + + + diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java new file mode 100644 index 0000000..33ddc10 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -0,0 +1,243 @@ +package com.jd.platform.jlog.nacos; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.AbstractSharedListener; +import com.alibaba.nacos.api.exception.NacosException; + +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.alibaba.nacos.api.common.Constants.DEFAULT_GROUP; +import static com.jd.platform.jlog.nacos.NacosConstant.*; + + +/** + * The type Nacos configuration. + * + * @author slievrly + */ +public class NacosConfigurator implements Configurator { + + private static volatile NacosConfigurator instance; + + private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + private static volatile ConfigService configService; + + static final ConcurrentMap> CONFIG_LISTENERS_MAP = new ConcurrentHashMap<>(8); + + static volatile Properties props = new Properties(); + + + public static NacosConfigurator getInstance() { + if (instance == null) { + synchronized (NacosConfigurator.class) { + if (instance == null) { + instance = new NacosConfigurator(); + } + } + } + return instance; + } + + + private NacosConfigurator() { + if (configService == null) { + try { + configService = NacosFactory.createConfigService(getConfigProperties()); + initConfig(); + } catch (NacosException e) { + throw new RuntimeException(e); + } + } + } + + + @Override + public String getConfig(String key) { + return null; + } + + @Override + public String getConfig(String key, long timeoutMills) { + String value = System.getProperty(key); + if (value != null) { + return value; + } + + value = props.getProperty(key); + + if (null == value) { + try { + value = configService.getConfig(key, DEFAULT_GROUP, timeoutMills); + } catch (NacosException exx) { + LOGGER.error(exx.getErrMsg()); + } + } + + return value; + } + + @Override + public boolean putConfig(String key, String content) { + return false; + } + + @Override + public boolean putConfig(String dataId, String content, long timeoutMills) { + boolean result = false; + try { + if (!props.isEmpty()) { + props.setProperty(dataId, content); + result = configService.publishConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, getConfigStr()); + } else { + result = configService.publishConfig(dataId, DEFAULT_GROUP, content); + } + } catch (NacosException exx) { + LOGGER.error(exx.getErrMsg()); + } + return result; + } + + + @Override + public boolean removeConfig(String dataId, long timeoutMills) { + boolean result = false; + try { + if (!props.isEmpty()) { + props.remove(dataId); + result = configService.publishConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, getConfigStr()); + } else { + result = configService.removeConfig(dataId, DEFAULT_GROUP); + } + } catch (NacosException exx) { + LOGGER.error(exx.getErrMsg()); + } + return result; + } + + @Override + public boolean removeConfig(String key) { + return false; + } + + + + @Override + public void addConfigListener(String dataId, ConfigChangeListener listener) { + if (StringUtils.isBlank(dataId) || listener == null) { + return; + } + try { + NacosListener nacosListener = new NacosListener(dataId, listener); + CONFIG_LISTENERS_MAP.computeIfAbsent(dataId, key -> new ConcurrentHashMap<>()) + .put(listener, nacosListener); + configService.addListener(dataId, DEFAULT_GROUP, nacosListener); + } catch (Exception exx) { + LOGGER.error("add nacos listener error:{}", exx.getMessage(), exx); + } + } + + @Override + public void removeConfigListener(String dataId, ConfigChangeListener listener) { + if (StringUtils.isBlank(dataId) || listener == null) { + return; + } + Set configChangeListeners = getConfigListeners(dataId); + if (configChangeListeners != null && configChangeListeners.size() > 0) { + for (ConfigChangeListener entry : configChangeListeners) { + if (listener.equals(entry)) { + NacosListener nacosListener = null; + Map configListeners = CONFIG_LISTENERS_MAP.get(dataId); + if (configListeners != null) { + nacosListener = configListeners.get(listener); + configListeners.remove(entry); + } + if (nacosListener != null) { + configService.removeListener(dataId, DEFAULT_GROUP, nacosListener); + } + break; + } + } + } + } + + @Override + public Set getConfigListeners(String dataId) { + Map configListeners = CONFIG_LISTENERS_MAP.get(dataId); + if (configListeners != null && configListeners.size() > 0) { + return configListeners.keySet(); + } else { + return null; + } + } + + + private static Properties getConfigProperties() { + Properties properties = new Properties(); + properties.setProperty(PRO_NAMESPACE_KEY, DEFAULT_NAMESPACE); + String address = FILE_CONFIG.getConfig("serverAddr",2000L); + if (address != null) { + properties.setProperty(PRO_SERVER_ADDR_KEY, address); + } + return properties; + } + + + + private static String getConfigStr() { + StringBuilder sb = new StringBuilder(); + + Enumeration enumeration = props.propertyNames(); + while (enumeration.hasMoreElements()) { + String key = (String) enumeration.nextElement(); + String property = props.getProperty(key); + sb.append(key).append("=").append(property).append("\n"); + } + + return sb.toString(); + } + + private static void initConfig() { + try { + String config = configService.getConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, 2000L); + if (StringUtils.isNotBlank(config)) { + try (Reader reader = new InputStreamReader(new ByteArrayInputStream(config.getBytes()), StandardCharsets.UTF_8)) { + props.load(reader); + } + NacosListener nacosListener = new NacosListener(DEFAULT_DATA_ID, null); + configService.addListener(DEFAULT_DATA_ID, DEFAULT_GROUP, nacosListener); + } + } catch (NacosException | IOException e) { + LOGGER.error("init config properties error", e); + } + } + + @Override + public String getType() { + return CONFIG_TYPE; + } + + @Override + public Map getConfigByPrefix(String prefix) { + return null; + } + + +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java new file mode 100644 index 0000000..f50b3f3 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java @@ -0,0 +1,12 @@ +package com.jd.platform.jlog.nacos; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + +public class NacosConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + return NacosConfigurator.getInstance(); + } +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java new file mode 100644 index 0000000..cbcb7b6 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java @@ -0,0 +1,20 @@ +package com.jd.platform.jlog.nacos; + + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosConstant.java + * @Description TODO + * @createTime 2022年02月22日 10:22:00 + */ +public class NacosConstant { + + static final String DEFAULT_GROUP = "JLOG_GROUP"; + static final String DEFAULT_DATA_ID = "jLog.properties"; + static final String PRO_SERVER_ADDR_KEY = "serverAddr"; + static final String CONFIG_TYPE = "nacos"; + static final String DEFAULT_NAMESPACE = "JLOG"; + static final String PRO_NAMESPACE_KEY = "namespace"; + static final int MAP_INITIAL_CAPACITY = 8; +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java new file mode 100644 index 0000000..35406e8 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -0,0 +1,86 @@ +package com.jd.platform.jlog.nacos; + +import com.alibaba.nacos.api.config.listener.AbstractSharedListener; +import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentMap; + +import static com.jd.platform.jlog.nacos.NacosConfigurator.CONFIG_LISTENERS_MAP; +import static com.jd.platform.jlog.nacos.NacosConfigurator.props; +import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; + +public class NacosListener extends AbstractSharedListener { + private final String dataId; + private final ConfigChangeListener listener; + + /** + * Instantiates a new Nacos listener. + * + * @param dataId the data id + * @param listener the listener + */ + public NacosListener(String dataId, ConfigChangeListener listener) { + this.dataId = dataId; + this.listener = listener; + } + + /** + * Gets target listener. + * + * @return the target listener + */ + public ConfigChangeListener getTargetListener() { + return this.listener; + } + + @Override + public void innerReceive(String dataId, String group, String configInfo) { + //The new configuration method to puts all configurations into a dateId + if (DEFAULT_DATA_ID.equals(dataId)) { + Properties seataConfigNew = new Properties(); + if (StringUtils.isNotBlank(configInfo)) { + try (Reader reader = new InputStreamReader(new ByteArrayInputStream(configInfo.getBytes()), StandardCharsets.UTF_8)) { + seataConfigNew.load(reader); + } catch (IOException e) { + // LOGGER.error("load config properties error", e); + return; + } + } + + //Get all the monitored dataids and judge whether it has been modified + for (Map.Entry> entry : CONFIG_LISTENERS_MAP.entrySet()) { + String listenedDataId = entry.getKey(); + String propertyOld = props.getProperty(listenedDataId, ""); + String propertyNew = seataConfigNew.getProperty(listenedDataId, ""); + if (!propertyOld.equals(propertyNew)) { + ConfigChangeEvent event = new ConfigChangeEvent() + .setKey(listenedDataId) + .setNewValue(propertyNew) + .setNamespace(group); + + ConcurrentMap configListeners = entry.getValue(); + for (ConfigChangeListener configListener : configListeners.keySet()) { + configListener.onProcessEvent(event); + } + } + } + + props = seataConfigNew; + return; + } + + //Compatible with old writing + ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo) + .setNamespace(group); + listener.onProcessEvent(event); + } + } diff --git a/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000..f442534 --- /dev/null +++ b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.nacos.NacosConfigurationProvider \ No newline at end of file diff --git a/config/config-zk/pom.xml b/config/config-zk/pom.xml new file mode 100644 index 0000000..93e9f4b --- /dev/null +++ b/config/config-zk/pom.xml @@ -0,0 +1,41 @@ + + + + + com.jd.platfrom.jlog + config + 1.4-SNAPSHOT + + 4.0.0 + config-zk + config-zk + + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT + + + org.apache.curator + curator-recipes + 4.2.0 + + + + diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java new file mode 100644 index 0000000..7bfae66 --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -0,0 +1,194 @@ +package com.jd.platform.jlog.zk; + +import java.io.ByteArrayInputStream; +import java.util.Enumeration; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.*; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.*; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.jd.platform.jlog.zk.ZkConstant.DEFAULT_CONFIG_PATH; +import static com.jd.platform.jlog.zk.ZkConstant.SERVER_ADDR_KEY; + + +/** + * @author didi + */ +public class ZkConfigurator implements Configurator { + + private final static Logger LOGGER = LoggerFactory.getLogger(ZkConfigurator.class); + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + static volatile CuratorFramework zkClient; + + static final ConcurrentMap CONFIG_LISTENERS_MAP = new ConcurrentHashMap<>(8); + static volatile Properties pros = new Properties(); + + + public ZkConfigurator() throws Exception { + if (zkClient == null) { + synchronized (ZkConfigurator.class) { + zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,1000L)) + // 连接超时时间 + .sessionTimeoutMs(2000) + // 会话超时时间 + .connectionTimeoutMs(6000) + .namespace("jLog") + // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + zkClient.start(); + } + LOGGER.info("初始化ZK,准备载入Zk数据到缓存"); + loadZkData(); + LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", pros.toString()); + } + } + + + @Override + public String getConfig(String key) { + String value = pros.getProperty(key); + if (value != null) { + return value; + } + + value = System.getProperty(key); + if (value != null) { + return value; + } + try { + return new String(zkClient.getData().forPath(DEFAULT_CONFIG_PATH)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + + @Override + public String getConfig(String key, long timeoutMills) { + return getConfig(key); + } + + + @Override + public boolean putConfig(String key, String content) { + if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ + return false; + } + pros.setProperty(key, content); + return true; + } + + + @Override + public boolean putConfig(String key, String content, long timeoutMills) { + pros.setProperty(key, content); + try { + zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigStr()); + } catch (Exception e) { + e.printStackTrace(); + } + return true; + } + + + @Override + public boolean removeConfig(String dataId, long timeoutMills) { + pros.remove(dataId); + try { + zkClient.delete().forPath(DEFAULT_CONFIG_PATH); + } catch (Exception e) { + e.printStackTrace(); + } + return true; + } + + @Override + public boolean removeConfig(String key) { + return false; + } + + + + @Override + public void addConfigListener(String key) { + LOGGER.info("ZK添加监听器, key:{}", key); + if (StringUtil.isBlank(key)) { + return; + } + ZkListener zkListener = new ZkListener("/"+key); + CONFIG_LISTENERS_MAP.put(key, zkListener); + // zkListener.onProcessEvent(new ConfigChangeEvent()); + LOGGER.info("ZK添加监听器之后, CONFIG_LISTENERS_MAP:{}", JSON.toJSONString(CONFIG_LISTENERS_MAP)); + } + + + @Override + public void removeConfigListener(String key) { + if (StringUtil.isBlank(key)) { + return; + } + LOGGER.info("ZK删除监听器, key:{}", key); + ConfigChangeListener configChangeListeners = getConfigListeners(key); + CONFIG_LISTENERS_MAP.remove(key); + ZkListener zkListener = (ZkListener) configChangeListeners; + zkListener.onShutDown(); + } + + + @Override + public ConfigChangeListener getConfigListeners(String key) { + return CONFIG_LISTENERS_MAP.get(key); + } + + + @Override + public Map getConfigByPrefix(String prefix) { + return null; + } + + @Override + public String getType() { + return "zk"; + } + + + + + + static void loadZkData() throws Exception { + + byte[] bt = zkClient.getData().forPath(DEFAULT_CONFIG_PATH); + if (bt != null && bt.length > 0){ + ByteArrayInputStream bArray = new ByteArrayInputStream(bt); + pros.load(bArray); + } + } + + + private static byte[] formatConfigStr() { + StringBuilder sb = new StringBuilder(); + + Enumeration enumeration = pros.propertyNames(); + while (enumeration.hasMoreElements()) { + String key = (String) enumeration.nextElement(); + String property = pros.getProperty(key); + sb.append(key).append("=").append(property).append("\n"); + } + LOGGER.info("ZK更新配置文件:{}", sb.toString()); + return sb.toString().getBytes(); + } + + +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java new file mode 100644 index 0000000..4f959e0 --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java @@ -0,0 +1,17 @@ +package com.jd.platform.jlog.zk; + + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorProvider; + + +public class ZkConfiguratorProvider implements ConfiguratorProvider { + @Override + public Configurator build() { + try { + return new ZkConfigurator(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java new file mode 100644 index 0000000..5792bfa --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java @@ -0,0 +1,18 @@ +package com.jd.platform.jlog.zk; + +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.apache.curator.framework.CuratorFramework; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ZkConstant.java + * @Description TODO + * @createTime 2022年02月22日 10:16:00 + */ +public class ZkConstant { + + static final String SERVER_ADDR_KEY = "serverAddr"; + static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java new file mode 100644 index 0000000..284c0e8 --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -0,0 +1,93 @@ +package com.jd.platform.jlog.zk; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.core.ConfigChangeEvent; +import com.jd.platform.jlog.core.ConfigChangeListener; +import com.jd.platform.jlog.core.ConfigChangeType; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +import static com.jd.platform.jlog.zk.ZkConfigurator.*; + + +/** + * @author didi + */ +public class ZkListener implements ConfigChangeListener { + + private final static Logger LOGGER = LoggerFactory.getLogger(ZkListener.class); + + private NodeCache cache; + + public ZkListener(String path) { + cache = new NodeCache(zkClient, path); + LOGGER.info("构造ZkListener:{}",path); + try { + cache.start(true); + } catch (Exception e) { + System.out.println("构造ZkListenereeeee"+e.getLocalizedMessage()); + e.printStackTrace(); + } + cache.getListenable().addListener(() -> { + String value = null; + if(null!=cache.getCurrentData()){ + value = new String( cache.getCurrentData().getData()); + } + System.out.println("=####====== "+value); + }); + System.out.println("=####= SIZE===== "+cache.getListenable().size()); + } + + + + @Override + public void onShutDown(){ + LOGGER.info("ZK删除监听器, 开始的监听器list:{}",JSON.toJSONString( cache.getListenable())); + try { + cache.close(); + } catch (IOException e) { + e.printStackTrace(); + } + LOGGER.info("ZK删除监听器, 完成的监听器list:{}",JSON.toJSONString( cache.getListenable())); + } + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + LOGGER.info("ZK数据变更,通用事件触发onChangeEvent",event.toString()); + // changeEvent(); + } + + + + private void changeEvent(){ + ChildData data = cache.getCurrentData(); + LOGGER.info("变更-当前数据cache:{}", cache); + Properties propsTmp = pros; + try { + LOGGER.info("ZK数据变更,旧Properties:{}", JSON.toJSONString(propsTmp)); + loadZkData(); + LOGGER.info("ZK数据变更,新Properties:{}", JSON.toJSONString(pros)); + } catch (Exception e) { + e.printStackTrace(); + } + LOGGER.info("ZK数据变更,CONFIG_LISTENERS_MAP:{}", JSON.toJSONString(CONFIG_LISTENERS_MAP)); + for (Map.Entry entry : CONFIG_LISTENERS_MAP.entrySet()) { + String listenedKey = entry.getKey(); + String propertyOld = pros.getProperty(listenedKey); + String propertyNew = propsTmp.getProperty(listenedKey); + if (!propertyOld.equals(propertyNew)) { + ConfigChangeEvent event = new ConfigChangeEvent() + .setKey(listenedKey) + .setNewValue(propertyNew) + .setChangeType(ConfigChangeType.MODIFY); + entry.getValue().onProcessEvent(event); + } + } + } +} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java new file mode 100644 index 0000000..15039e7 --- /dev/null +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java @@ -0,0 +1,63 @@ +package com.jd.platform.jlog.zk; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.jd.platform.jlog.zk.ZkConstant.SERVER_ADDR_KEY; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ZkTest.java + * @Description TODO + * @createTime 2022年02月22日 17:26:00 + */ +public class ZkTest { + private final static Logger log = LoggerFactory.getLogger(ZkTest.class); + + + public static void main(String[] args) throws Exception { + CuratorFramework client = CuratorFrameworkFactory.builder().connectString("101.42.242.201:2181") + // 连接超时时间 + .sessionTimeoutMs(2000) + // 会话超时时间 + .connectionTimeoutMs(6000) + // .namespace("jLog") + // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + //开启客户端 + client.start(); + //创建缓存节点 + NodeCache nodeCache = new NodeCache(client, "/jLog/jLog.properties"); + + //将该节点数据初始化到本地缓存 + nodeCache.start(true); + //添加节点监听事件,NodeCacheListener每次都会触发，但不能获取监听的操作类型到底是添加还是删除等。 + nodeCache.getListenable().addListener(() -> { + String value = null; + System.out.println("Listener-name: "+Thread.currentThread().getName()); + if(null!=nodeCache.getCurrentData()){ + value = new String(nodeCache.getCurrentData().getData()); + } + System.out.println("======= "+value); + }); + + + client.setData().forPath("/jLog/jLog.properties","noweeee".getBytes()); + //睡眠等待监听事件触发 + Thread.sleep(25000); + System.out.println("main-name: "+Thread.currentThread().getName()); + + //关闭 + nodeCache.close(); + client.close(); + + } + +} diff --git a/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider new file mode 100644 index 0000000..f8a41f9 --- /dev/null +++ b/config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -0,0 +1 @@ +com.jd.platform.jlog.zk.ZkConfiguratorProvider \ No newline at end of file diff --git a/config/pom.xml b/config/pom.xml new file mode 100644 index 0000000..e9885c4 --- /dev/null +++ b/config/pom.xml @@ -0,0 +1,36 @@ + + + + + JLog + com.jd.platfrom.jlog + 1.4-SNAPSHOT + + 4.0.0 + config + pom + config + + config-core + config-etcd + config-apollo + config-nacos + config-zk + + diff --git a/pom.xml b/pom.xml index 57bd6e2..eb3fca9 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ clientlogback Dashboard clientdemo + config diff --git a/worker/pom.xml b/worker/pom.xml index d1a06a8..8e02fe6 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -22,6 +22,11 @@ org.springframework.boot spring-boot-starter-web + + com.jd.platfrom.jlog + config-zk + 1.4-SNAPSHOT + org.springframework.boot diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java index 8835f51..d716012 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.worker.config; -import com.jd.platform.jlog.common.config.IConfigCenter; import com.jd.platform.jlog.common.constant.Constant; import com.jd.platform.jlog.common.utils.IpUtils; +import com.jd.platform.jlog.core.ConfiguratorFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -35,11 +35,7 @@ public class CenterStarter { */ @Value("${config.mdc}") private String mdc; - /** - * configCenter - */ - @Resource - private IConfigCenter configCenter; + /** * 上报自己的ip到配置中心 @@ -50,10 +46,8 @@ public class CenterStarter { scheduledExecutorService.scheduleAtFixedRate(() -> { try { - //configCenter.putAndGrant(buildKey(), buildValue(), 8); - //configCenter.putAndGrant(buildSecondKey(), buildValue(), 8); - configCenter.put(buildKey(), buildValue()); - configCenter.put(buildSecondKey(), buildValue()); + ConfiguratorFactory.getInstance().putConfig(buildKey(), buildValue()); + ConfiguratorFactory.getInstance().putConfig(buildSecondKey(), buildValue()); } catch (Exception e) { //do nothing e.printStackTrace(); diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java index 3e7d533..21d33b5 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java @@ -1,17 +1,10 @@ package com.jd.platform.jlog.worker.config; -import com.jd.platform.jlog.common.config.ConfigCenterEnum; -import com.jd.platform.jlog.common.config.ConfigCenterFactory; -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.model.CenterConfig; -import com.jd.platform.jlog.common.model.TagConfig; -import com.jd.platform.jlog.common.utils.ConfigUtil; + +import com.jd.platform.jlog.common.tag.TagConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; @@ -26,17 +19,8 @@ public class ConfigCenter { private Logger logger = LoggerFactory.getLogger(getClass()); - private CenterConfig centerConfig; private TagConfig tagConfig ; - public CenterConfig getCenterConfig() { - return centerConfig; - } - - public void setCenterConfig(CenterConfig centerConfig) { - this.centerConfig = centerConfig; - } - public TagConfig getTagConfig() { return tagConfig; } @@ -45,13 +29,4 @@ public class ConfigCenter { this.tagConfig = tagConfig; } - @Bean - public IConfigCenter client() throws Exception { - - ConfigCenterFactory.buildConfigCenter(centerConfig); - //连接多个时，逗号分隔 - //return JdEtcdBuilder.build(etcdServer); - return ConfigCenterFactory.getClient(ConfigUtil.getCenter(centerConfig)); - } - } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java b/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java index 7167995..98ebaf7 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.worker.controller; -import cn.hutool.core.date.DateUtil; import com.jd.platform.jlog.worker.db.Db; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -32,12 +31,12 @@ public class UserController { Map map0 = new HashMap<>(); map0.put("clientType", 1); map0.put("pin", "abcd"); - map0.put("createTime", DateUtil.formatDateTime(new Date())); + // map0.put("createTime", DateUtil.formatDateTime(new Date())); Map map1 = new HashMap<>(); map1.put("clientType", 1); map1.put("pin", "abcd"); - map1.put("createTime", DateUtil.formatDateTime(new Date())); + // map1.put("createTime", DateUtil.formatDateTime(new Date())); datas.add(map0); datas.add(map1); diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java index 7469555..5ac5061 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.worker.disruptor; -import cn.hutool.core.date.DateUtil; import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.model.TracerData; @@ -105,7 +104,7 @@ public class TracerConsumer implements WorkHandler { map.put("threadName", runLogMessage.getThreadName()); map.put("methodName", runLogMessage.getMethodName()); map.put("logLevel", runLogMessage.getLogLevel()); - map.put("createTime", DateUtil.formatDateTime(new Date(runLogMessage.getCreateTime()))); + // map.put("createTime", DateUtil.formatDateTime(new Date(runLogMessage.getCreateTime()))); map.put("content", runLogMessage.getContent()); tracerLogToDbStore.offer(map); } @@ -138,7 +137,7 @@ public class TracerConsumer implements WorkHandler { //此处做了一个base64编码，否则原编码直接进去，取出来后是String，直接getBytes后无法用Zstd解压 map.put("responseContent", responseBytes); - map.put("createTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); + // map.put("createTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); map.put("costTime", tracerBean.getCostTime()); @@ -179,7 +178,7 @@ public class TracerConsumer implements WorkHandler { String serverIp = requestMap.get("serverIp") == null ? "" : requestMap.get("serverIp").toString(); map.put("serverIp", serverIp); - map.put("intoDbTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); + // map.put("intoDbTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); tracerModelToDbStore.offer(map); } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java index e226b6d..fa8b5fc 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.worker.store; -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.collect.Queues; + import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.AsyncWorker; import com.jd.platform.jlog.worker.db.Db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,8 +94,8 @@ public class TracerLogToDbStore { try { List> tempModels = new ArrayList<>(); //每1s入库一次 - Queues.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempModels)) { + AsyncWorker.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS); + if (tempModels.size() == 0) { continue; } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java index 4ad9d8f..8a54550 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.worker.store; -import cn.hutool.core.collection.CollectionUtil; import com.google.common.collect.Queues; import com.jd.platform.jlog.common.utils.AsyncPool; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.worker.db.Db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -- Gitee From b422fac77d7bdec1b453ce230779fd4cbefcda03 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Tue, 22 Feb 2022 19:28:31 +0800 Subject: [PATCH 10/22] =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=99=A8=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/common/config/etcd/JdEtcdClient.java | 71 ------------------- .../jlog/common/config/nacos/NacosClient.java | 68 ------------------ 2 files changed, 139 deletions(-) delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java deleted file mode 100644 index 9dbac30..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.jd.platform.jlog.common.config.etcd; - -import cn.hutool.core.collection.CollectionUtil; -import com.google.protobuf.ByteString; -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.RangeResponse; -import com.ibm.etcd.client.EtcdClient; -import com.ibm.etcd.client.KvStoreClient; -import com.ibm.etcd.client.kv.KvClient; -import com.ibm.etcd.client.lease.LeaseClient; -import com.ibm.etcd.client.lock.LockClient; -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.model.CenterConfig; - -import java.util.List; -import java.util.Map; - -/** - * etcd客户端 - * - * @author wuweifeng wrote on 2019-12-06 - * @version 1.0 - */ -public class JdEtcdClient implements IConfigCenter { - - - private KvClient kvClient; - private LeaseClient leaseClient; - private LockClient lockClient; - - public JdEtcdClient() {} - - public JdEtcdClient(KvStoreClient kvStoreClient) { - this.kvClient = kvStoreClient.getKvClient(); - this.leaseClient = kvStoreClient.getLeaseClient(); - this.lockClient = kvStoreClient.getLockClient(); - } - - - @Override - public IConfigCenter buildClient(CenterConfig config){ - String etcdServer = config.getEtcdServer(); - return new JdEtcdClient(EtcdClient.forEndpoints(etcdServer).withPlainText().build()); - } - - @Override - public void put(String key, String value) { - kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value)).sync(); - } - - @Override - public String get(String key) { - RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync(); - List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { - return null; - } - return keyValues.get(0).getValue().toStringUtf8(); - } - - @Override - public void delete(String key) { - - } - - @Override - public List getPrefixKey(String key) { - return null; - } - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java b/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java deleted file mode 100644 index e970fd5..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/nacos/NacosClient.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.jd.platform.jlog.common.config.nacos; - -import com.alibaba.nacos.api.NacosFactory; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.exception.NacosException; -import com.jd.platform.jlog.common.config.IConfigCenter; -import com.jd.platform.jlog.common.model.CenterConfig; - -import java.util.List; -import java.util.Map; -import java.util.Properties; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName NacosClient.java - * @Description TODO - * @createTime 2022年02月10日 21:47:00 - */ -public class NacosClient implements IConfigCenter { - - - private ConfigService configService; - - public NacosClient(){} - - public NacosClient(ConfigService configService){ - this.configService = configService; - } - - - @Override - public IConfigCenter buildClient(CenterConfig config) throws NacosException { - String serverAddr = config.getNacosServer(); - Properties properties = new Properties(); - properties.put("serverAddr", serverAddr); - ConfigService configService = NacosFactory.createConfigService(properties); - return new NacosClient(configService); - } - - @Override - public void put(String key, String value){ - try { - boolean result = configService.publishConfig("data1", "group1", "nacosContentTEXT"); - System.out.println("publishConfig result==-> "+result); - } catch (Exception e) { - System.out.println("==-> "+e.toString()); - e.printStackTrace(); - } - } - - @Override - public String get(String key) throws NacosException { - return configService.getConfig("data1", "group1", 5000); - } - - @Override - public void delete(String key) { - - } - - @Override - public List getPrefixKey(String key) { - return null; - } - - -} -- Gitee From 2995002284472adce4f569b4953be0c28339844c Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Fri, 25 Feb 2022 00:15:55 +0800 Subject: [PATCH 11/22] =?UTF-8?q?=E9=80=82=E9=85=8D=E5=A4=9A=E7=A7=8D?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E7=9B=91=E5=90=AC=E5=AE=8C=E6=88=90=EF=BC=8C?= =?UTF-8?q?=20todo=20tag=E5=8D=87=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 21 ++- .../jlog/client/TracerClientStarter.java | 22 ++- .../platform/jlog/client/udp/UdpClient.java | 4 +- .../platform/jlog/client/udp/UdpSender.java | 1 + .../appender/TracerLogbackAppender.java | 6 +- .../src/main/resources/application.properties | 6 +- clientdemo/src/main/resources/application.yml | 2 +- .../jlog/common/config/ConfigCenterEnum.java | 15 -- .../common/config/ConfigCenterFactory.java | 58 -------- .../jlog/common/config/IConfigCenter.java | 49 ------- .../jlog/common/constant/Constant.java | 10 +- .../jlog/common/model/RunLogMessage.java | 15 ++ .../jlog/common/model/TracerBean.java | 2 +- .../platform/jlog/common/tag/CollectMode.java | 73 ++++++++++ .../com/jd/platform/jlog/common/tag/Tag.java | 25 ++++ .../platform/jlog/common/tag/TagConfig.java | 61 +++++++- .../platform/jlog/common/tag/TagHandler.java | 25 +++- config/config-apollo/pom.xml | 10 +- .../config/apollo/ApolloConfigurator.java | 47 +++--- .../apollo/ApolloConfiguratorProvider.java | 5 +- .../jlog/config/apollo/ApolloConstant.java | 5 +- .../jlog/config/apollo/ApolloListener.java | 5 +- .../platform/jlog/core/ConfigChangeEvent.java | 67 ++------- .../jlog/core/ConfigChangeListener.java | 16 ++- .../platform/jlog/core/ConfigChangeType.java | 3 + .../jd/platform/jlog/core/Configurator.java | 7 +- .../jlog/core/ConfiguratorFactory.java | 19 +++ .../jlog/core/ConfiguratorProvider.java | 4 +- .../platform/jlog/core/FileConfigurator.java | 44 +++--- .../jlog/etcd/EtcdConfigurationProvider.java | 3 + .../platform/jlog/etcd/EtcdConfigurator.java | 42 ++++-- .../jd/platform/jlog/etcd/EtcdListener.java | 28 ++-- config/config-nacos/pom.xml | 2 +- .../jlog/nacos/NacosConfigurator.java | 97 ++++++------- .../jd/platform/jlog/nacos/NacosConstant.java | 6 +- .../jd/platform/jlog/nacos/NacosListener.java | 134 +++++++++--------- .../com/jd/platform/jlog/nacos/NacosTest.java | 68 +++++++++ ...jd.platform.jlog.core.ConfiguratorProvider | 2 +- .../jd/platform/jlog/zk/ZkConfigurator.java | 12 +- .../com/jd/platform/jlog/zk/ZkConstant.java | 6 +- .../com/jd/platform/jlog/zk/ZkListener.java | 9 +- 41 files changed, 614 insertions(+), 422 deletions(-) delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java create mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java diff --git a/client/pom.xml b/client/pom.xml index 4da6548..1cfd9f8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,12 +18,29 @@ 1.4-SNAPSHOT - + + + + + + com.jd.platfrom.jlog + config-apollo + 1.4-SNAPSHOT + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index e433d37..3fc028a 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.client; import com.alibaba.fastjson.JSON; +import com.ctrip.framework.apollo.ConfigChangeListener; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.client.udp.HttpSender; @@ -12,7 +13,6 @@ import com.jd.platform.jlog.common.tag.TagHandler; import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; -import com.jd.platform.jlog.zk.ZkConfigurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +25,7 @@ import org.slf4j.LoggerFactory; */ public class TracerClientStarter { - private final static Logger LOGGER = LoggerFactory.getLogger(ZkConfigurator.class); + private final static Logger LOGGER = LoggerFactory.getLogger(TracerClientStarter.class); /** @@ -89,6 +89,7 @@ public class TracerClientStarter { checkAndSetTagConfig(); TagHandler.build(tagConfig); + Context.MDC = mdc; Monitor starter = new Monitor(); @@ -109,11 +110,11 @@ public class TracerClientStarter { * 如果未赋值，从配置器获取，底层是Properties */ private void checkAndSetTagConfig(){ + Configurator configurator = ConfiguratorFactory.getInstance(); if(tagConfig != null){ LOGGER.info("从主配置获取的tagConfig", tagConfig.toString()); return; } - Configurator configurator = ConfiguratorFactory.getInstance(); String reqTag = configurator.getConfig("reqTags"); String logTag = configurator.getConfig("logTags"); String regex = configurator.getConfig("regex"); @@ -123,8 +124,17 @@ public class TracerClientStarter { .logTags(FastJsonUtils.toList(logTag, String.class)) .regex(regex).delimiter(delimiter).join(join).build(); LOGGER.info("从配置器获取的tagConfig:{}", tagConfig.toString()); - configurator.addConfigListener("jLog.properties"); - configurator.removeConfigListener("jLog.properties"); - + System.out.println("添加监听器开始1111"); + configurator.addConfigListener("/jLog/properties"); + System.out.println("添加监听器开始2222"); + configurator.addConfigListener("a"); + + try { + Thread.sleep(20000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println("移除监听器开始"); + configurator.removeConfigListener("/jLog/properties"); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java index e947662..af4645b 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java @@ -60,7 +60,7 @@ public class UdpClient { }); //4.bind并返回一个channel try { - Channel channel = bootstrap.bind(8888).sync().channel(); + Channel channel = bootstrap.bind(8887).sync().channel(); Context.CHANNEL = channel; //6.等待channel的close @@ -68,6 +68,8 @@ public class UdpClient { //7.关闭group group.shutdownGracefully(); } catch (InterruptedException e) { + System.out.println("NioEventLoopGroup ==> "+e.toString()); + e.printStackTrace(); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java index c563181..90081d4 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java @@ -144,6 +144,7 @@ public class UdpSender { private static void send(List tracerBeans) { TracerData tracerData = new TracerData(); tracerData.setTracerBeanList(tracerBeans); + System.out.println("Context.CHANNEL==> "+Context.CHANNEL); Context.CHANNEL.writeAndFlush(tracerData); } } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java index 234466f..6db352c 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -61,10 +61,8 @@ public class TracerLogbackAppender extends AppenderBase { String formattedMessage = getMessage(loggingEvent); logMessage.setContent(formattedMessage); - /* Map map = TagHandler.extractLogTag(formattedMessage); - if(map != null){ - System.out.println("Logback tag map =>> "+ JSON.toJSONString(map)); - }*/ + Map map = TagHandler.extractLogTag(formattedMessage); + return logMessage; } diff --git a/clientdemo/src/main/resources/application.properties b/clientdemo/src/main/resources/application.properties index 8d51d0c..4da1f28 100644 --- a/clientdemo/src/main/resources/application.properties +++ b/clientdemo/src/main/resources/application.properties @@ -1 +1,5 @@ -server.port=8082 \ No newline at end of file +server.port=8083 + +apollo.meta=http://127.0.0.1:8080 +apollo.config-service=http://127.0.0.1:8080 +app.id=order \ No newline at end of file diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml index d4ffa15..69f332c 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/clientdemo/src/main/resources/application.yml @@ -1,4 +1,4 @@ -serverAddr: 101.42.242.201:2181 +serverAddr: 127.0.0.1:2379 jlog: center-config: center: etcd diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java deleted file mode 100644 index e6120cc..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterEnum.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.jd.platform.jlog.common.config; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName ConfigCenterEnum.java - * @Description TODO - * @createTime 2022年02月10日 20:32:00 - */ -public enum ConfigCenterEnum { - APOLLO, - NACOS, - ETCD, - ZK, -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java b/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java deleted file mode 100644 index 6a7e4c3..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/ConfigCenterFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.jd.platform.jlog.common.config; - - -import com.google.common.collect.Maps; -import com.jd.platform.jlog.common.config.apollo.ApolloClient; -import com.jd.platform.jlog.common.config.etcd.JdEtcdClient; -import com.jd.platform.jlog.common.config.nacos.NacosClient; -import com.jd.platform.jlog.common.config.zookeeper.ZKClient; -import com.jd.platform.jlog.common.model.CenterConfig; -import com.sun.deploy.util.ReflectionUtil; - -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Map; - -import static com.jd.platform.jlog.common.utils.ConfigUtil.getCenter; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName ConfigCenterFactory.java - * @Description TODO - * @createTime 2022年02月10日 20:31:00 - */ -public class ConfigCenterFactory { - - private static Map configCenterMap = Maps.newHashMap(); - - - public static void buildConfigCenter(CenterConfig config) throws Exception { - - ConfigCenterEnum center = getCenter(config); - - switch (center){ - case APOLLO: - configCenterMap.put(center, new ApolloClient().buildClient(config)); - break; - case NACOS: - configCenterMap.put(center,new NacosClient().buildClient(config)); - break; - case ETCD: - configCenterMap.put(center, new JdEtcdClient().buildClient(config)); - break; - case ZK: - configCenterMap.put(center, new ZKClient().buildClient(config)); - break; - default: - System.out.println("Unsupported config center"); - } - } - - - public static IConfigCenter getClient(ConfigCenterEnum center) { - return configCenterMap.get(center); - } - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java b/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java deleted file mode 100644 index 1ef1333..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jd.platform.jlog.common.config; - -import com.alibaba.nacos.api.exception.NacosException; -import com.jd.platform.jlog.common.model.CenterConfig; - -import java.util.List; -import java.util.Map; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName AbsConfigCenter.java - * @Description TODO - * @createTime 2022年02月10日 20:33:00 - */ -public interface IConfigCenter { - - - /** - * 构建config-center client - * @param config - * @throws Exception - */ - IConfigCenter buildClient(CenterConfig config) throws Exception; - - /** - * 存入key，value - */ - void put(String key, String value); - - /** - * 根据key，获取value - */ - String get(String key) throws NacosException; - - - /** - * 删除 - * @param key - */ - void delete(String key); - - - /** - * 获取指定前缀的所有key - */ - List getPrefixKey(String key); - -} diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index 9429a1b..8673867 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -29,11 +29,19 @@ public class Constant { public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; - + /** + * 普通tag最大长度 超过则不会单独存储，不超过则会存入tag：{"normal" : "logContent"} + */ public static int TAG_NORMAL_KEY_MAX_LEN = 20; + /** + * 符合正则但不构成kv结构的普通日志，如「xx查询为空」 + */ public static String TAG_NORMAL_KEY = "normal"; + /** + * 正则里需要特殊处理的字符串 + */ public static final Set SPECIAL_CHAR = new HashSet() { { add("*"); diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java index 665c5ad..6af7ab7 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java @@ -1,5 +1,7 @@ package com.jd.platform.jlog.common.model; +import java.util.Map; + /** * className：RunLogMessage * description： @@ -37,6 +39,11 @@ public class RunLogMessage { */ private String threadName; + /** + * 标签map + */ + private Map tagMap; + public long getTracerId() { return tracerId; } @@ -92,4 +99,12 @@ public class RunLogMessage { public void setThreadName(String threadName) { this.threadName = threadName; } + + public Map getTagMap() { + return tagMap; + } + + public void setTagMap(Map tagMap) { + this.tagMap = tagMap; + } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java index 66b2b2f..ac6b03d 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java @@ -25,7 +25,7 @@ public class TracerBean implements Serializable { private String tracerId; /** * tracer对象，里面放的是List> - * 第一个元素是request对象，key为request，value为完整request入参，从中可找到pin、uuid + * 第一个元素是request对象，key为request，value为完整request入参，从中可找到uuid * 最后一个元素是response响应，key为response，value为响应值byte[]，可转为String入库 */ private List> tracerObject; diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java new file mode 100644 index 0000000..5e1646c --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java @@ -0,0 +1,73 @@ +package com.jd.platform.jlog.common.tag; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CollectMode.java + * @Description 定义采集模式的组合 用于以后method级别的tag注解 + * @createTime 2022年02月24日 23:10:00 + */ + +public class CollectMode { + + /** + * 提取req + */ + public static final long E_REQ = 1L; + + /** + * 压缩req + */ + public static final long C_REQ = 1L << 2; + + /** + * 提取log + */ + public static final long E_LOG = 1L << 3; + + /** + * 压缩log + */ + public static final long C_LOG = 1L << 4; + + /** + * 提取log + */ + public static final long E_RESP = 1L << 5; + + /** + * 压缩log + */ + public static final long C_RESP = 1L << 6; + + + + /** ======================================= 下面是组合 ======================================= */ + + /** + * 提取req，压缩req + */ + public static final long E_REQ_C_REQ = E_REQ | C_REQ; + + /** + * 提取log，压缩log + */ + public static final long E_LOG_C_LOG = E_LOG | C_LOG; + + /** + * 提取req+log，压缩req + */ + public static final long E_REQ_E_LOG_C_REQ = E_REQ | E_LOG | C_REQ; + + /** + * 提取req+log，压缩log + */ + public static final long E_REQ_E_LOG_C_LOG = E_REQ | E_LOG | C_LOG; + + /** + * 提取req+log，压缩req+log + */ + public static final long E_REQ_E_LOG_C_REQ_E_LOG = E_REQ | E_LOG | C_REQ | E_LOG; + + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java b/common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java new file mode 100644 index 0000000..72fc759 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java @@ -0,0 +1,25 @@ +package com.jd.platform.jlog.common.tag; + +import java.lang.annotation.*; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Tag.java + * @Description TODO 后续完善使用 + * @createTime 2022年02月24日 20:21:00 + */ +@Documented +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Tag { + + boolean extractReq(); + + boolean compressReq(); + + boolean extractResp(); + + boolean compressResp(); + +} \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java index 5b92ccf..c71d20a 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java @@ -11,16 +11,42 @@ import java.util.List; */ public class TagConfig implements Serializable { + /** + * 入参的tag + */ private List reqTags; + /** + * 普通日志的tag + */ private List logTags; + /** + * 自定义正则 可为空 + */ private String regex; + /** + * 分隔符 默认｜ + */ private String delimiter = "|"; + /** + * 连接符 默认= + */ private String join = "="; + /** + * 提取入参开关 + */ + private Boolean extractReq = true; + + /** + * 提取普通log开关 + */ + private Boolean extractLog = true; + + public List getReqTags() { return reqTags; } @@ -61,13 +87,30 @@ public class TagConfig implements Serializable { this.join = join; } + public Boolean getExtractReq() { + return extractReq; + } + + public void setExtractReq(Boolean extractReq) { + this.extractReq = extractReq; + } + + public Boolean getExtractLog() { + return extractLog; + } + + public void setExtractLog(Boolean extractLog) { + this.extractLog = extractLog; + } public static final class Builder { private List reqTags; private List logTags; private String regex; - private String delimiter = "|"; - private String join = "="; + private String delimiter; + private String join; + private Boolean extractReq; + private Boolean extractLog; public Builder() { } @@ -101,6 +144,16 @@ public class TagConfig implements Serializable { return this; } + public Builder extractReq(Boolean extractReq) { + this.extractReq = extractReq; + return this; + } + + public Builder extractLog(Boolean extractLog) { + this.extractLog = extractLog; + return this; + } + public TagConfig build() { TagConfig tagConfig = new TagConfig(); tagConfig.setReqTags(reqTags); @@ -108,6 +161,8 @@ public class TagConfig implements Serializable { tagConfig.setRegex(regex); tagConfig.setDelimiter(delimiter); tagConfig.setJoin(join); + tagConfig.setExtractReq(extractReq); + tagConfig.setExtractLog(extractLog); return tagConfig; } } @@ -120,6 +175,8 @@ public class TagConfig implements Serializable { ", regex='" + regex + '\'' + ", delimiter='" + delimiter + '\'' + ", join='" + join + '\'' + + ", extractReq='" + extractReq + '\'' + + ", extractLog='" + extractLog + '\'' + '}'; } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java index cf4c819..f037e63 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java @@ -28,8 +28,12 @@ public class TagHandler { private Set reqTags; + private boolean extractReq; + private Set logTags; + private boolean extractLog; + private String delimiter = "|"; private int delimiterLen = delimiter.length(); @@ -40,17 +44,24 @@ public class TagHandler { private static volatile TagHandler INSTANCE = null; - /** * 构建标签处理器 * @param tagConfig 配置类 */ public static void build(TagConfig tagConfig) { + if(!tagConfig.getExtractReq() && !tagConfig.getExtractLog()){ + return; + } + TagHandler handler = new TagHandler(); - String regex = tagConfig.getRegex(); + + handler.extractReq = tagConfig.getExtractReq(); + handler.extractLog = tagConfig.getExtractLog(); handler.reqTags = new HashSet<>(tagConfig.getReqTags()); handler.logTags = new HashSet<>(tagConfig.getLogTags()); + + String regex = tagConfig.getRegex(); if(StringUtil.isNotEmpty(regex)){ handler.pattern = Pattern.compile(regex); }else{ @@ -62,7 +73,7 @@ public class TagHandler { handler.delimiterLen = tagConfig.getDelimiter().length(); handler.join = tagConfig.getJoin(); INSTANCE = handler; - //LOGGER.info("构建标签处理器单例完成:{}",INSTANCE.toString()); + LOGGER.info("构建标签处理器单例完成:{}",INSTANCE.toString()); } @@ -74,7 +85,7 @@ public class TagHandler { */ public static Map extractReqTag(Map params, @NotNull Map ext) { - if(INSTANCE == null){ return null; } + if(INSTANCE == null || !INSTANCE.extractReq){ return null; } Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); for (String tag : INSTANCE.reqTags) { @@ -99,7 +110,7 @@ public class TagHandler { * @return tags */ public static Map extractLogTag(String content) { - if(INSTANCE == null || content.length() < 1){ + if(INSTANCE == null || !INSTANCE.extractLog || content.length() < 1){ return null; } @@ -126,10 +137,10 @@ public class TagHandler { /** - * 刷新标签处理器 + * 刷新标签处理器 加锁是为了防止极端情况下, 先到的config1覆盖后到的config2 * @param tagConfig 新的配置 */ - public static void refresh(TagConfig tagConfig) { + public synchronized static void refresh(TagConfig tagConfig) { INSTANCE = null; build(tagConfig); } diff --git a/config/config-apollo/pom.xml b/config/config-apollo/pom.xml index 5156749..dc41155 100644 --- a/config/config-apollo/pom.xml +++ b/config/config-apollo/pom.xml @@ -15,12 +15,6 @@ com.jd.platfrom.jlog config-core 1.4-SNAPSHOT - - - slf4j-api - org.slf4j - - com.ctrip.framework.apollo @@ -31,10 +25,10 @@ slf4j-api org.slf4j - + diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index df9361a..46b3e2d 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -2,27 +2,39 @@ package com.jd.platform.jlog.config.apollo; import java.util.Map; import java.util.Properties; -import java.util.Set; import java.util.concurrent.*; import com.ctrip.framework.apollo.core.utils.StringUtils; import com.ctrip.framework.apollo.enums.PropertyChangeType; import com.ctrip.framework.apollo.model.ConfigChange; -import com.ctrip.framework.apollo.spring.property.AutoUpdateConfigChangeListener; import com.jd.platform.jlog.core.*; import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.ConfigService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import static com.ctrip.framework.apollo.core.ApolloClientSystemConsts.APP_ID; import static com.jd.platform.jlog.config.apollo.ApolloConstant.*; /** - * @author didi + * @author tangbohu + * @version 1.0.0 + * @Description todo env and cluster + * @ClassName ApolloConfigurator.java + * @createTime 2022年02月21日 21:21:00 */ public class ApolloConfigurator implements Configurator { + + private static final Logger LOGGER = LoggerFactory.getLogger(ApolloConfigurator.class); + + private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + /** + * 里面有resourceProperties 和 configProperties + */ private static volatile Config config; private static final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(); private static volatile ApolloConfigurator instance; @@ -32,10 +44,13 @@ public class ApolloConfigurator implements Configurator { if (config == null) { synchronized (ApolloConfigurator.class) { if (config == null) { - config = ConfigService.getConfig(FILE_CONFIG.getConfig(DEFAULT_NAMESPACE)); + // apollo的监听是按照namespace维度 + config = ConfigService.getConfig(DEFAULT_NAMESPACE); config.addChangeListener(changeEvent -> { + LOGGER.info("Apollo收到事件变更, keys={}", changeEvent.changedKeys()); for (String key : changeEvent.changedKeys()) { if (!CONFIG_LISTENER_MAP.containsKey(key)) { + LOGGER.info("{}不是关注的key,直接返回", key); continue; } ConfigChange change = changeEvent.getChange(key); @@ -47,13 +62,10 @@ public class ApolloConfigurator implements Configurator { } } } + System.out.println("Apollo配置器构建完成"); } - /** - * Gets instance. - * - * @return the instance - */ + public static ApolloConfigurator getInstance() { if (instance == null) { synchronized (ApolloConfigurator.class) { @@ -67,19 +79,15 @@ public class ApolloConfigurator implements Configurator { @Override public String getConfig(String key) { - return null; + return config.getProperty(key,""); } @Override public String getConfig(String key, long timeoutMills) { - String value = System.getProperty(key); - if (value != null) { - return value; - } - - return config.getProperty(key,""); + return getConfig(key,1L); } + @Override public boolean putConfig(String key, String content) { return false; @@ -112,8 +120,10 @@ public class ApolloConfigurator implements Configurator { CONFIG_LISTENER_MAP.put(key, new ApolloListener()); } + @Override public void removeConfigListener(String key) { + System.out.println("Apollo进入移除"+key); CONFIG_LISTENER_MAP.remove(key); } @@ -149,6 +159,7 @@ public class ApolloConfigurator implements Configurator { private void loadApolloServerConfig() { + Properties properties = System.getProperties(); if (!properties.containsKey(PROP_APP_ID)) { System.setProperty(PROP_APP_ID, FILE_CONFIG.getConfig(APP_ID)); @@ -163,7 +174,9 @@ public class ApolloConfigurator implements Configurator { } } if (!properties.containsKey(APOLLO_CLUSTER)) { - System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getConfig(APOLLO_CLUSTER)); + if (!StringUtils.isBlank(FILE_CONFIG.getConfig(APOLLO_CLUSTER))) { + System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getConfig(APOLLO_CLUSTER)); + } } if (!properties.containsKey(APOLLO_CONFIG_SERVICE)) { System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getConfig(APOLLO_CONFIG_SERVICE)); diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java index 9288be1..6fedf35 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java @@ -4,7 +4,10 @@ import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorProvider; /** - * @author didi + * @author tangbohu + * @version 1.0.0 + * @ClassName ApolloConfiguratorProvider.java + * @createTime 2022年02月21日 21:26:00 */ public class ApolloConfiguratorProvider implements ConfiguratorProvider { @Override diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java index 39833b4..9861134 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java @@ -4,7 +4,6 @@ package com.jd.platform.jlog.config.apollo; * @author tangbohu * @version 1.0.0 * @ClassName ApolloConstant.java - * @Description TODO * @createTime 2022年02月22日 11:06:00 */ public class ApolloConstant { @@ -12,10 +11,10 @@ public class ApolloConstant { static final String APOLLO_META = "apolloMeta"; static final String APOLLO_SECRET = "apolloAccesskeySecret"; static final String APOLLO_CLUSTER = "jLog"; - static final String APOLLO_CONFIG_SERVICE = "apolloConfigService"; + static final String APOLLO_CONFIG_SERVICE = "apollo.config-service"; static final String PROP_APP_ID = "app.id"; static final String PROP_APOLLO_META = "apollo.meta"; - static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.configService"; + static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.config-service"; static final String PROP_APOLLO_SECRET = "apollo.accesskey.secret"; static final String PROP_APOLLO_CLUSTER = "apollo.cluster"; static final String DEFAULT_NAMESPACE = "jLog"; diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java index fb1b268..cbcccdc 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java @@ -7,15 +7,12 @@ import com.jd.platform.jlog.core.ConfigChangeListener; * @author tangbohu * @version 1.0.0 * @ClassName ApolloListener.java - * @Description TODO * @createTime 2022年02月22日 19:18:00 */ public class ApolloListener implements ConfigChangeListener { - @Override public void onChangeEvent(ConfigChangeEvent event) { - - + LOGGER.info("APOLLO 重写的事件 event={}", event.toString()); } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java index c9113f3..301b35a 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -2,7 +2,10 @@ package com.jd.platform.jlog.core; /** - * @author didi + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigChangeEvent.java + * @createTime 2022年02月16日 22:13:00 */ public class ConfigChangeEvent { @@ -11,7 +14,7 @@ public class ConfigChangeEvent { private String newValue; private String namespace; private ConfigChangeType changeType; - private static final String DEFAULT_NAMESPACE = "DEFAULT"; + private static final String DEFAULT_NAMESPACE = "jLog"; public ConfigChangeEvent(){ @@ -31,96 +34,56 @@ public class ConfigChangeEvent { this.changeType = type; } - /** - * Gets data id. - * - * @return the data id - */ + public String getKey() { return key; } - /** - * Sets data id. - * - * @param key key - */ + public ConfigChangeEvent setKey(String key) { this.key = key; return this; } - /** - * Gets old value. - * - * @return the old value - */ + public String getOldValue() { return oldValue; } - /** - * Sets old value. - * - * @param oldValue the old value - */ + public ConfigChangeEvent setOldValue(String oldValue) { this.oldValue = oldValue; return this; } - /** - * Gets new value. - * - * @return the new value - */ + public String getNewValue() { return newValue; } - /** - * Sets new value. - * - * @param newValue the new value - */ + public ConfigChangeEvent setNewValue(String newValue) { this.newValue = newValue; return this; } - /** - * Gets change type. - * - * @return the change type - */ + public ConfigChangeType getChangeType() { return changeType; } - /** - * Sets change type. - * - * @param changeType the change type - */ + public ConfigChangeEvent setChangeType(ConfigChangeType changeType) { this.changeType = changeType; return this; } - /** - * Gets namespace. - * - * @return the namespace - */ + public String getNamespace() { return namespace; } - /** - * Sets namespace. - * - * @param namespace the namespace - */ + public ConfigChangeEvent setNamespace(String namespace) { this.namespace = namespace; return this; diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java index 4e8c81f..83994aa 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -16,14 +16,21 @@ package com.jd.platform.jlog.core; -import java.util.concurrent.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.concurrent.*; /** - * @author didi + * @author tangbohu + * @version 1.0.0 + * @ClassName ConfigChangeListener.java + * @createTime 2022年02月16日 12:06:00 */ public interface ConfigChangeListener { + Logger LOGGER = LoggerFactory.getLogger(ConfigChangeListener.class); + ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(1, 1, Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), Executors.defaultThreadFactory()); @@ -32,7 +39,12 @@ public interface ConfigChangeListener { void onChangeEvent(ConfigChangeEvent event); + /** + * 处理变更事件 + * @param event 事件 + */ default void onProcessEvent(ConfigChangeEvent event) { + LOGGER.info("通用监听器收到了变更事件, event={}", event.toString()); getExecutorService().submit(() -> { beforeEvent(); try { diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java index a799103..acdee80 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java @@ -1,6 +1,9 @@ package com.jd.platform.jlog.core; +/** + * @author tangbohu + */ public enum ConfigChangeType { ADD, diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index 45cae7e..c2829f6 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -4,6 +4,10 @@ import java.util.Map; /** * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName Configurator.java + * @createTime 2022年02月15日 17:06:00 */ public interface Configurator { @@ -25,6 +29,7 @@ public interface Configurator { /** * 设置配置 * @param key key + * @param content val * @return content val */ boolean putConfig(String key, String content); @@ -83,7 +88,7 @@ public interface Configurator { /** * 根据前缀/父级路径获取子节点 - * @param prefix + * @param prefix path * @return List */ Map getConfigByPrefix(String prefix); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java index 664f130..ca5b759 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -25,6 +25,10 @@ import java.util.ServiceLoader; /** * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName ConfiguratorFactory.java + * @createTime 2022年02月15日 21:54:00 */ public class ConfiguratorFactory { @@ -51,14 +55,25 @@ public class ConfiguratorFactory { return instance; } + + /** + * SPI实现装载不同的配置器 + * @return 配置器 + */ private static Configurator buildConfiguration() { try { + /** + * 配置文件的配置器 + */ base = new FileConfigurator(); } catch (IOException e) { e.printStackTrace(); } + /** + * 配置中心的配置器，如果没有就用文件配置器 + */ ServiceLoader builders = ServiceLoader.load(ConfiguratorProvider.class); LOGGER.info("ServiceLoader获取到的实现类:{}", builders.toString()); for (ConfiguratorProvider provider : builders) { @@ -68,6 +83,10 @@ public class ConfiguratorFactory { return base; } + /** + * 重载 + * @throws Exception + */ protected static void reload() throws Exception { instance = null; getInstance(); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java index 82b7e59..f1ebb13 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java @@ -7,8 +7,8 @@ package com.jd.platform.jlog.core; public interface ConfiguratorProvider { /** - * - * @return + * 根据实现类构建配置器 + * @return 配置器 */ Configurator build(); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index 4fa22ab..fe5b330 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -12,12 +12,18 @@ import java.util.concurrent.*; /** - * @author didi + * @author tangbohu + * @version 1.0.0 + * @ClassName FileConfigurator.java + * @createTime 2022年02月17日 23:22:00 */ public class FileConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class); + /** + * 监听重读配置文件间隔 单位ms + */ private static final long LISTENER_CONFIG_INTERVAL = 5000; private static final String CONFIG_FILE_PROPERTIES = "/application.properties"; @@ -26,9 +32,12 @@ public class FileConfigurator implements Configurator { private static final String JLOG_CONFIG_FILE = "/jLog.properties"; + /** + * 配置文件集合 + */ private static final String[] FILE_ARRAY = { CONFIG_FILE_PROPERTIES, CONFIG_FILE_YML, JLOG_CONFIG_FILE }; - private final ConcurrentMap configListenerMap = new ConcurrentHashMap<>(8); + private final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(8); private final Map listenedConfigMap = new HashMap<>(8); @@ -74,7 +83,7 @@ public class FileConfigurator implements Configurator { } if(properties.size() == 0){ - // 不正常 warn + // warn return null; } return properties.getProperty(key); @@ -103,7 +112,7 @@ public class FileConfigurator implements Configurator { return; } FileListener fileListener = new FileListener(); - configListenerMap.put(key, fileListener); + CONFIG_LISTENER_MAP.put(key, fileListener); listenedConfigMap.put(key, ConfiguratorFactory.getInstance().getConfig(key,1000L)); fileListener.addListener(key, fileListener); } @@ -119,7 +128,7 @@ public class FileConfigurator implements Configurator { @Override public void removeConfigListener(String key) { ConfigChangeListener configListener = getConfigListeners(key); - configListenerMap.remove(key); + CONFIG_LISTENER_MAP.remove(key); listenedConfigMap.remove(key); configListener.onShutDown(); } @@ -128,7 +137,7 @@ public class FileConfigurator implements Configurator { @Override public ConfigChangeListener getConfigListeners(String key) { - return configListenerMap.get(key); + return CONFIG_LISTENER_MAP.get(key); } @@ -147,7 +156,6 @@ public class FileConfigurator implements Configurator { class FileListener implements ConfigChangeListener { - private final Map> keyListenersMap = new HashMap<>(); private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), @@ -155,18 +163,18 @@ public class FileConfigurator implements Configurator { FileListener() {} - public synchronized void addListener(String dataId, ConfigChangeListener listener) { - if (keyListenersMap.isEmpty()) { + public synchronized void addListener(String key, ConfigChangeListener listener) { + if (CONFIG_LISTENER_MAP.isEmpty()) { fileListener.onProcessEvent(new ConfigChangeEvent()); } - keyListenersMap.computeIfAbsent(dataId, value -> new HashSet<>()).add(listener); + CONFIG_LISTENER_MAP.put(key, listener); } @Override public void onChangeEvent(ConfigChangeEvent event) { while (true) { - for (String key : keyListenersMap.keySet()) { + for (String key : CONFIG_LISTENER_MAP.keySet()) { try { checkAndConfigure(); String currentConfig = properties.getProperty(key); @@ -175,14 +183,11 @@ public class FileConfigurator implements Configurator { if (currentConfig.equals(oldConfig)) { listenedConfigMap.put(key, currentConfig); event.setKey(key).setNewValue(currentConfig).setOldValue(oldConfig); - - for (ConfigChangeListener listener : keyListenersMap.get(key)) { - listener.onChangeEvent(event); - } + CONFIG_LISTENER_MAP.get(key).onChangeEvent(event); } } - } catch (Exception exx) { - LOGGER.error("fileListener execute error, key :{}", key, exx); + } catch (Exception e) { + LOGGER.error("fileListener execute error, key :{}", key, e); } } try { @@ -200,8 +205,9 @@ public class FileConfigurator implements Configurator { } - - + /** + * 检测文件最后修改时间 和重载文件 + */ private void checkAndConfigure() { File file; diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java index 61d198b..09a5e2e 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java @@ -4,6 +4,9 @@ package com.jd.platform.jlog.etcd; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorProvider; +/** + * @author tangbohu + */ public class EtcdConfigurationProvider implements ConfiguratorProvider { @Override public Configurator build() { diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index 57c5b1c..deb48d0 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -1,25 +1,27 @@ package com.jd.platform.jlog.etcd; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; + +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArraySet; +import com.alibaba.fastjson.JSON; import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.EtcdClient; import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.StringUtil; import com.jd.platform.jlog.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * - * @author didi + * @author tangbohu + * @version 1.0.0 + * @desc 参考log4J + * @ClassName EtcdConfigurator.java + * @createTime 2022年02月21日 21:46:00 */ public class EtcdConfigurator implements Configurator { @@ -30,13 +32,26 @@ public class EtcdConfigurator implements Configurator { static volatile EtcdClient client; private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; - private static final String SERVER_ADDR_KEY = "serverAddr"; + private ConcurrentMap configListenerMap = new ConcurrentHashMap<>(8); + private static final String SERVER_ADDR_KEY = "serverAddr"; + + private static final String ROOT = "/jLog"; + + private static final String PROPERTIES_PATH = "/properties"; + + private static Properties PROPERTIES = new Properties(); + + private EtcdConfigurator() { LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); client = EtcdClient.forEndpoints(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,2000L)).withPlainText().build(); + String val = getConfig(ROOT + PROPERTIES_PATH); + if(StringUtil.isNotBlank(val)){ + PROPERTIES.putAll((Map)JSON.parse(val)); + } } @@ -87,18 +102,19 @@ public class EtcdConfigurator implements Configurator { @Override public boolean removeConfig(String key) { client.getKvClient().delete(ByteString.copyFromUtf8(key)).sync(); - return false; + return true; } @Override public boolean removeConfig(String key, long timeoutMills) { client.getKvClient().delete(ByteString.copyFromUtf8(key)).timeout(timeoutMills).sync(); - return false; + return true; } @Override public void addConfigListener(String key) { + System.out.println("添加etcd监听器"+key); EtcdListener etcdListener = new EtcdListener(key); configListenerMap.put(key, etcdListener); etcdListener.onProcessEvent(new ConfigChangeEvent()); @@ -106,6 +122,7 @@ public class EtcdConfigurator implements Configurator { @Override public void removeConfigListener(String key) { + System.out.println("移除etcd监听器"+key); ConfigChangeListener configListener = getConfigListeners(key); configListenerMap.remove(key); configListener.onShutDown(); @@ -113,8 +130,8 @@ public class EtcdConfigurator implements Configurator { @Override - public ConfigChangeListener getConfigListeners(String dataId) { - return configListenerMap.get(dataId); + public ConfigChangeListener getConfigListeners(String key) { + return configListenerMap.get(key); } @@ -123,6 +140,7 @@ public class EtcdConfigurator implements Configurator { return "etcd"; } + @Override public Map getConfigByPrefix(String prefix) { RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(prefix)).asPrefix().sync(); diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index 2b9d734..882794e 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -31,6 +31,21 @@ public class EtcdListener implements ConfigChangeListener { public EtcdListener(String key) { this.key = key; + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(key)).start(); + System.out.println("构造器EtcdListener"); + + getExecutorService().submit(() -> { + while (iterator.hasNext()){ + Event eve = iterator.next().getEvents().get(0); + KeyValue kv = eve.getKv(); + Event.EventType eveType = eve.getType(); + ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; + + ConfigChangeEvent event = new ConfigChangeEvent(); + event.setKey(key).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + onChangeEvent(event); + } + }); } @@ -40,21 +55,16 @@ public class EtcdListener implements ConfigChangeListener { getExecutorService().shutdownNow(); } + @Override public void onChangeEvent(ConfigChangeEvent event) { - iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(key)).start(); - while (iterator.hasNext()){ - Event eve = iterator.next().getEvents().get(0); - KeyValue kv = eve.getKv(); - Event.EventType eveType = eve.getType(); - ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; - event.setKey(key).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); - this.onChangeEvent(event); - } + System.out.println("onChangeEvent 一次又一次的进入 ==> "+event.getKey()); } @Override public ExecutorService getExecutorService() { return executor; } + + } diff --git a/config/config-nacos/pom.xml b/config/config-nacos/pom.xml index 5878cbf..911115e 100644 --- a/config/config-nacos/pom.xml +++ b/config/config-nacos/pom.xml @@ -20,7 +20,7 @@ com.alibaba.nacos nacos-client - 2.0.4 + 1.4.2 diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java index 33ddc10..9740d7b 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -9,20 +9,18 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import com.alibaba.fastjson.JSON; import com.alibaba.nacos.api.NacosFactory; import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.config.listener.AbstractSharedListener; import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.common.utils.StringUtils; -import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.alibaba.nacos.api.common.Constants.DEFAULT_GROUP; import static com.jd.platform.jlog.nacos.NacosConstant.*; @@ -33,14 +31,17 @@ import static com.jd.platform.jlog.nacos.NacosConstant.*; */ public class NacosConfigurator implements Configurator { + private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); + + private static volatile NacosConfigurator instance; - private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + private static volatile ConfigService configService; - static final ConcurrentMap> CONFIG_LISTENERS_MAP = new ConcurrentHashMap<>(8); + static final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(8); static volatile Properties props = new Properties(); @@ -61,6 +62,7 @@ public class NacosConfigurator implements Configurator { if (configService == null) { try { configService = NacosFactory.createConfigService(getConfigProperties()); + LOGGER.info("实例化NacosConfigurator完成 result = {}",configService.getServerStatus()); initConfig(); } catch (NacosException e) { throw new RuntimeException(e); @@ -85,7 +87,7 @@ public class NacosConfigurator implements Configurator { if (null == value) { try { - value = configService.getConfig(key, DEFAULT_GROUP, timeoutMills); + value = configService.getConfig(key, JLOG_GROUP, timeoutMills); } catch (NacosException exx) { LOGGER.error(exx.getErrMsg()); } @@ -105,9 +107,9 @@ public class NacosConfigurator implements Configurator { try { if (!props.isEmpty()) { props.setProperty(dataId, content); - result = configService.publishConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, getConfigStr()); + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, getConfigStr()); } else { - result = configService.publishConfig(dataId, DEFAULT_GROUP, content); + result = configService.publishConfig(dataId, JLOG_GROUP, content); } } catch (NacosException exx) { LOGGER.error(exx.getErrMsg()); @@ -122,9 +124,9 @@ public class NacosConfigurator implements Configurator { try { if (!props.isEmpty()) { props.remove(dataId); - result = configService.publishConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, getConfigStr()); + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, getConfigStr()); } else { - result = configService.removeConfig(dataId, DEFAULT_GROUP); + result = configService.removeConfig(dataId, JLOG_GROUP); } } catch (NacosException exx) { LOGGER.error(exx.getErrMsg()); @@ -132,67 +134,52 @@ public class NacosConfigurator implements Configurator { return result; } + @Override public boolean removeConfig(String key) { return false; } - @Override - public void addConfigListener(String dataId, ConfigChangeListener listener) { - if (StringUtils.isBlank(dataId) || listener == null) { - return; - } + public void addConfigListener(String dataId) { + LOGGER.info("nacos添加监听器开始 dataId:{}",dataId); + NacosListener nacosListener = new NacosListener(dataId); + CONFIG_LISTENER_MAP.put(dataId, nacosListener); + LOGGER.info("## nacos添加监听器过程 {}",nacosListener.toString()); + try { - NacosListener nacosListener = new NacosListener(dataId, listener); - CONFIG_LISTENERS_MAP.computeIfAbsent(dataId, key -> new ConcurrentHashMap<>()) - .put(listener, nacosListener); - configService.addListener(dataId, DEFAULT_GROUP, nacosListener); - } catch (Exception exx) { - LOGGER.error("add nacos listener error:{}", exx.getMessage(), exx); + configService.addListener(dataId, JLOG_GROUP, nacosListener); + } catch (NacosException e) { + LOGGER.error("nacos添加监听器失败",e); } } + + @Override - public void removeConfigListener(String dataId, ConfigChangeListener listener) { - if (StringUtils.isBlank(dataId) || listener == null) { + public void removeConfigListener(String dataId) { + if (StringUtils.isBlank(dataId)) { return; } - Set configChangeListeners = getConfigListeners(dataId); - if (configChangeListeners != null && configChangeListeners.size() > 0) { - for (ConfigChangeListener entry : configChangeListeners) { - if (listener.equals(entry)) { - NacosListener nacosListener = null; - Map configListeners = CONFIG_LISTENERS_MAP.get(dataId); - if (configListeners != null) { - nacosListener = configListeners.get(listener); - configListeners.remove(entry); - } - if (nacosListener != null) { - configService.removeListener(dataId, DEFAULT_GROUP, nacosListener); - } - break; - } - } - } + LOGGER.info("nacos移除监听器开始"); + ConfigChangeListener changeListener = getConfigListeners(dataId); + NacosListener listener = (NacosListener) changeListener; + LOGGER.info("## nacos移除监听器过程 {}",listener.toString()); + + CONFIG_LISTENER_MAP.remove(dataId); + configService.removeListener(dataId, JLOG_GROUP, listener); } @Override - public Set getConfigListeners(String dataId) { - Map configListeners = CONFIG_LISTENERS_MAP.get(dataId); - if (configListeners != null && configListeners.size() > 0) { - return configListeners.keySet(); - } else { - return null; - } + public ConfigChangeListener getConfigListeners(String dataId) { + return CONFIG_LISTENER_MAP.get(dataId); } private static Properties getConfigProperties() { Properties properties = new Properties(); - properties.setProperty(PRO_NAMESPACE_KEY, DEFAULT_NAMESPACE); - String address = FILE_CONFIG.getConfig("serverAddr",2000L); + String address = FILE_CONFIG.getConfig(PRO_SERVER_ADDR_KEY); if (address != null) { properties.setProperty(PRO_SERVER_ADDR_KEY, address); } @@ -216,13 +203,16 @@ public class NacosConfigurator implements Configurator { private static void initConfig() { try { - String config = configService.getConfig(DEFAULT_DATA_ID, DEFAULT_GROUP, 2000L); + String config = configService.getConfig(DEFAULT_DATA_ID, JLOG_GROUP, 2000L); + LOGGER.info("从NaCos获取配置进行初始化 config = {}", config); + if (StringUtils.isNotBlank(config)) { try (Reader reader = new InputStreamReader(new ByteArrayInputStream(config.getBytes()), StandardCharsets.UTF_8)) { props.load(reader); } - NacosListener nacosListener = new NacosListener(DEFAULT_DATA_ID, null); - configService.addListener(DEFAULT_DATA_ID, DEFAULT_GROUP, nacosListener); + LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(props)); + NacosListener nacosListener = new NacosListener(DEFAULT_DATA_ID); + configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, nacosListener); } } catch (NacosException | IOException e) { LOGGER.error("init config properties error", e); @@ -231,9 +221,10 @@ public class NacosConfigurator implements Configurator { @Override public String getType() { - return CONFIG_TYPE; + return "nacos"; } + @Override public Map getConfigByPrefix(String prefix) { return null; diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java index cbcb7b6..326932c 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java @@ -10,11 +10,7 @@ package com.jd.platform.jlog.nacos; */ public class NacosConstant { - static final String DEFAULT_GROUP = "JLOG_GROUP"; + static final String JLOG_GROUP = "JLOG_GROUP"; static final String DEFAULT_DATA_ID = "jLog.properties"; static final String PRO_SERVER_ADDR_KEY = "serverAddr"; - static final String CONFIG_TYPE = "nacos"; - static final String DEFAULT_NAMESPACE = "JLOG"; - static final String PRO_NAMESPACE_KEY = "namespace"; - static final int MAP_INITIAL_CAPACITY = 8; } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java index 35406e8..d4415a3 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -1,86 +1,86 @@ package com.jd.platform.jlog.nacos; +import com.alibaba.fastjson.JSON; import com.alibaba.nacos.api.config.listener.AbstractSharedListener; +import com.alibaba.nacos.api.naming.listener.Event; +import com.alibaba.nacos.api.naming.listener.EventListener; import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; -import java.util.Map; +import java.io.*; import java.util.Properties; -import java.util.concurrent.ConcurrentMap; -import static com.jd.platform.jlog.nacos.NacosConfigurator.CONFIG_LISTENERS_MAP; + +import static com.jd.platform.jlog.nacos.NacosConfigurator.CONFIG_LISTENER_MAP; import static com.jd.platform.jlog.nacos.NacosConfigurator.props; import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; +import static com.jd.platform.jlog.nacos.NacosConstant.JLOG_GROUP; -public class NacosListener extends AbstractSharedListener { - private final String dataId; - private final ConfigChangeListener listener; - - /** - * Instantiates a new Nacos listener. - * - * @param dataId the data id - * @param listener the listener - */ - public NacosListener(String dataId, ConfigChangeListener listener) { - this.dataId = dataId; - this.listener = listener; - } - /** - * Gets target listener. - * - * @return the target listener - */ - public ConfigChangeListener getTargetListener() { - return this.listener; - } +public class NacosListener extends AbstractSharedListener implements ConfigChangeListener, EventListener { - @Override - public void innerReceive(String dataId, String group, String configInfo) { - //The new configuration method to puts all configurations into a dateId - if (DEFAULT_DATA_ID.equals(dataId)) { - Properties seataConfigNew = new Properties(); - if (StringUtils.isNotBlank(configInfo)) { - try (Reader reader = new InputStreamReader(new ByteArrayInputStream(configInfo.getBytes()), StandardCharsets.UTF_8)) { - seataConfigNew.load(reader); - } catch (IOException e) { - // LOGGER.error("load config properties error", e); - return; - } - } - //Get all the monitored dataids and judge whether it has been modified - for (Map.Entry> entry : CONFIG_LISTENERS_MAP.entrySet()) { - String listenedDataId = entry.getKey(); - String propertyOld = props.getProperty(listenedDataId, ""); - String propertyNew = seataConfigNew.getProperty(listenedDataId, ""); - if (!propertyOld.equals(propertyNew)) { - ConfigChangeEvent event = new ConfigChangeEvent() - .setKey(listenedDataId) - .setNewValue(propertyNew) - .setNamespace(group); - - ConcurrentMap configListeners = entry.getValue(); - for (ConfigChangeListener configListener : configListeners.keySet()) { - configListener.onProcessEvent(event); - } - } - } + private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); - props = seataConfigNew; - return; - } + private final String dataId; + + private final String group; + + + public NacosListener(String dataId) { + this.dataId = dataId; + this.group = JLOG_GROUP; + } + + @Override + public void innerReceive(String dataId, String group, String configInfo) { - //Compatible with old writing - ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo) - .setNamespace(group); - listener.onProcessEvent(event); + LOGGER.info("configInfo:{}", configInfo); + System.out.println("自带监听器" + configInfo); + if (DEFAULT_DATA_ID.equals(dataId)) { + Properties properties = new Properties(); + if (StringUtils.isNotBlank(configInfo)) { + try { + properties.load(new StringReader(configInfo)); + } catch (IOException e) { + return; + } + } + LOGGER.info("CONFIG_LISTENER_MAP :{}",JSON.toJSONString(CONFIG_LISTENER_MAP)); + ConfigChangeListener listener = CONFIG_LISTENER_MAP.get(dataId); + boolean same = CollectionUtil.equals(properties, props); + if(!same){ + ConfigChangeEvent event = new ConfigChangeEvent() + .setKey(DEFAULT_DATA_ID) + .setOldValue(JSON.toJSONString(props)) + .setNewValue(JSON.toJSONString(properties)) + .setNamespace(group); + listener.onProcessEvent(event); + props = properties; + } + return; } + + ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo).setNamespace(group); + this.onProcessEvent(event); + } + + + + @Override + public void onChangeEvent(ConfigChangeEvent event) { + System.out.println("通用[配置]变更事件"); + LOGGER.info("通用[配置]变更事件 event:{}",event); + } + + + @Override + public void onEvent(Event event) { + System.out.println("通用[服务]事件"); + LOGGER.info("通用[服务]事件 event:{}",event); } +} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java new file mode 100644 index 0000000..68e3218 --- /dev/null +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java @@ -0,0 +1,68 @@ +package com.jd.platform.jlog.nacos; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingFactory; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.listener.NamingEvent; + +import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; +import static com.jd.platform.jlog.nacos.NacosConstant.JLOG_GROUP; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosTest.java + * @Description TODO + * @createTime 2022年02月22日 22:04:00 + */ +public class NacosTest { + + public static void main(String[] args) throws NacosException, InterruptedException { + String serverAddr = "101.42.242.201:8848"; + String dataId = DEFAULT_DATA_ID; + String group = JLOG_GROUP; + ConfigService configService = NacosFactory.createConfigService(serverAddr); + + NamingService naming = NamingFactory.createNamingService(serverAddr); + naming.registerInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1"); + Thread.sleep(3000); + NacosListener server = new NacosListener(DEFAULT_DATA_ID); + naming.subscribe("nacos.test1", JLOG_GROUP,server); + String content = configService.getConfig(dataId, group, 2000L); + System.out.println("content: "+content); + NacosListener nL = new NacosListener(DEFAULT_DATA_ID); + configService.addListener(dataId, group, nL); + + System.out.println("新增完成"); + Thread.sleep(20000); + System.out.println("移除开始"); + configService.removeListener(dataId, group, nL); + System.out.println("移除结束"); + + naming.deregisterInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1"); + Thread.sleep(5000); + + System.out.println("睡眠结束"); + + /* System.out.println("移除开始222"); + configService.removeListener(dataId, group, new Listener() { + @Override + public Executor getExecutor() { + return Executors.newSingleThreadExecutor(); + } + + @Override + public void receiveConfigInfo(String s) { + System.out.println("#### 移除开始222"); + } + });*/ + + Thread.sleep(90000); + System.out.println("---------------------------睡眠结束"); + + } + + +} diff --git a/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider index f442534..6d982a3 100644 --- a/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider +++ b/config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider @@ -1 +1 @@ -com.jd.platform.jlog.nacos.NacosConfigurationProvider \ No newline at end of file +com.jd.platform.jlog.nacos.NacosConfiguratorProvider \ No newline at end of file diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java index 7bfae66..11ff738 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -16,6 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.jd.platform.jlog.zk.ZkConstant.DEFAULT_CONFIG_PATH; +import static com.jd.platform.jlog.zk.ZkConstant.NAMESPACE; import static com.jd.platform.jlog.zk.ZkConstant.SERVER_ADDR_KEY; @@ -37,20 +38,19 @@ public class ZkConfigurator implements Configurator { public ZkConfigurator() throws Exception { if (zkClient == null) { synchronized (ZkConfigurator.class) { - zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,1000L)) + zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getConfig(SERVER_ADDR_KEY)) // 连接超时时间 .sessionTimeoutMs(2000) // 会话超时时间 .connectionTimeoutMs(6000) - .namespace("jLog") + .namespace(NAMESPACE) // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 .retryPolicy(new ExponentialBackoffRetry(1000, 3)) .build(); zkClient.start(); } - LOGGER.info("初始化ZK,准备载入Zk数据到缓存"); loadZkData(); - LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", pros.toString()); + LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(pros)); } } @@ -129,8 +129,7 @@ public class ZkConfigurator implements Configurator { } ZkListener zkListener = new ZkListener("/"+key); CONFIG_LISTENERS_MAP.put(key, zkListener); - // zkListener.onProcessEvent(new ConfigChangeEvent()); - LOGGER.info("ZK添加监听器之后, CONFIG_LISTENERS_MAP:{}", JSON.toJSONString(CONFIG_LISTENERS_MAP)); + zkListener.onProcessEvent(new ConfigChangeEvent()); } @@ -166,7 +165,6 @@ public class ZkConfigurator implements Configurator { - static void loadZkData() throws Exception { byte[] bt = zkClient.getData().forPath(DEFAULT_CONFIG_PATH); diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java index 5792bfa..d0d2ba7 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java @@ -1,9 +1,5 @@ package com.jd.platform.jlog.zk; -import com.jd.platform.jlog.core.Configurator; -import com.jd.platform.jlog.core.ConfiguratorFactory; -import org.apache.curator.framework.CuratorFramework; - /** * @author tangbohu * @version 1.0.0 @@ -13,6 +9,8 @@ import org.apache.curator.framework.CuratorFramework; */ public class ZkConstant { + static final String NAMESPACE = "jLog"; + static final String ROOT = "/jLog"; static final String SERVER_ADDR_KEY = "serverAddr"; static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java index 284c0e8..29cc595 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -31,13 +31,12 @@ public class ZkListener implements ConfigChangeListener { try { cache.start(true); } catch (Exception e) { - System.out.println("构造ZkListenereeeee"+e.getLocalizedMessage()); e.printStackTrace(); } cache.getListenable().addListener(() -> { String value = null; if(null!=cache.getCurrentData()){ - value = new String( cache.getCurrentData().getData()); + value = new String(cache.getCurrentData().getData()); } System.out.println("=####====== "+value); }); @@ -48,23 +47,21 @@ public class ZkListener implements ConfigChangeListener { @Override public void onShutDown(){ - LOGGER.info("ZK删除监听器, 开始的监听器list:{}",JSON.toJSONString( cache.getListenable())); + LOGGER.info("ZK删除监听器"); try { cache.close(); } catch (IOException e) { e.printStackTrace(); } - LOGGER.info("ZK删除监听器, 完成的监听器list:{}",JSON.toJSONString( cache.getListenable())); } @Override public void onChangeEvent(ConfigChangeEvent event) { LOGGER.info("ZK数据变更,通用事件触发onChangeEvent",event.toString()); - // changeEvent(); + changeEvent(); } - private void changeEvent(){ ChildData data = cache.getCurrentData(); LOGGER.info("变更-当前数据cache:{}", cache); -- Gitee From 15f336e4dae7bb9e7d84a974d01870f1987a699a Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Sat, 26 Feb 2022 21:20:02 +0800 Subject: [PATCH 12/22] =?UTF-8?q?=E6=94=AF=E6=8C=81yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 7 +- .../jlog/client/TracerClientStarter.java | 19 +- .../platform/jlog/client/udp/UdpSender.java | 1 - .../jlog/clientdemo/config/DemoConfig.java | 2 +- .../src/main/resources/application.properties | 5 - clientdemo/src/main/resources/application.yml | 23 +- .../src/main/resources/application.ymlbak | 8 +- clientdemo/src/main/resources/jLog.properties | 1 + config/config-core/pom.xml | 5 + .../platform/jlog/core/ConfigChangeEvent.java | 4 + .../jlog/core/ConfigChangeListener.java | 4 +- .../jd/platform/jlog/core/Configurator.java | 12 +- .../jlog/core/ConfiguratorFactory.java | 1 - .../platform/jlog/core/FileConfigurator.java | 277 +++++++++++------- .../jd/platform/jlog/core/FileConstant.java | 43 +++ .../jd/platform/jlog/etcd/EtcdListener.java | 1 + 16 files changed, 260 insertions(+), 153 deletions(-) delete mode 100644 clientdemo/src/main/resources/application.properties create mode 100644 clientdemo/src/main/resources/jLog.properties create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java diff --git a/client/pom.xml b/client/pom.xml index 1cfd9f8..9486691 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -35,11 +35,16 @@ config-etcd 1.4-SNAPSHOT --> - + + + com.jd.platfrom.jlog + config-core + 1.4-SNAPSHOT javax.servlet diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index 3fc028a..68ca85d 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -1,8 +1,6 @@ package com.jd.platform.jlog.client; -import com.alibaba.fastjson.JSON; -import com.ctrip.framework.apollo.ConfigChangeListener; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.client.udp.HttpSender; @@ -88,7 +86,7 @@ public class TracerClientStarter { // 校验和设置 checkAndSetTagConfig(); - TagHandler.build(tagConfig); + // TagHandler.build(tagConfig); Context.MDC = mdc; @@ -112,7 +110,8 @@ public class TracerClientStarter { private void checkAndSetTagConfig(){ Configurator configurator = ConfiguratorFactory.getInstance(); if(tagConfig != null){ - LOGGER.info("从主配置获取的tagConfig", tagConfig.toString()); + LOGGER.info("从主配置获取的tagConfig:{}", tagConfig.toString()); + configurator.addConfigListener("/application.yml"); return; } String reqTag = configurator.getConfig("reqTags"); @@ -124,17 +123,5 @@ public class TracerClientStarter { .logTags(FastJsonUtils.toList(logTag, String.class)) .regex(regex).delimiter(delimiter).join(join).build(); LOGGER.info("从配置器获取的tagConfig:{}", tagConfig.toString()); - System.out.println("添加监听器开始1111"); - configurator.addConfigListener("/jLog/properties"); - System.out.println("添加监听器开始2222"); - configurator.addConfigListener("a"); - - try { - Thread.sleep(20000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - System.out.println("移除监听器开始"); - configurator.removeConfigListener("/jLog/properties"); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java index 90081d4..c563181 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java @@ -144,7 +144,6 @@ public class UdpSender { private static void send(List tracerBeans) { TracerData tracerData = new TracerData(); tracerData.setTracerBeanList(tracerBeans); - System.out.println("Context.CHANNEL==> "+Context.CHANNEL); Context.CHANNEL.writeAndFlush(tracerData); } } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java index d83830b..8fd2d29 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java +++ b/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java @@ -19,7 +19,7 @@ import javax.annotation.PostConstruct; * @date 2021-12-27 */ @Component -@ConfigurationProperties(prefix = "jlog") +@ConfigurationProperties() public class DemoConfig { private Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/clientdemo/src/main/resources/application.properties b/clientdemo/src/main/resources/application.properties deleted file mode 100644 index 4da1f28..0000000 --- a/clientdemo/src/main/resources/application.properties +++ /dev/null @@ -1,5 +0,0 @@ -server.port=8083 - -apollo.meta=http://127.0.0.1:8080 -apollo.config-service=http://127.0.0.1:8080 -app.id=order \ No newline at end of file diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml index 69f332c..2ec88a2 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/clientdemo/src/main/resources/application.yml @@ -1,11 +1,16 @@ -serverAddr: 127.0.0.1:2379 -jlog: - center-config: - center: etcd - etcdServer: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ - nacosServer: 101.42.242.201:8848 - zkServer: 101.42.242.201:2181 - +serverAddr: 127.0.0.1:111 +a: 3333 +tag-config: + reqTags: + - uid + - ip + logTags: + - errno + - node + regex: "" + delimiter: "|" + join: = server: - port: 8081 + port: 8085 + diff --git a/clientdemo/src/main/resources/application.ymlbak b/clientdemo/src/main/resources/application.ymlbak index 165f59c..e0a50af 100644 --- a/clientdemo/src/main/resources/application.ymlbak +++ b/clientdemo/src/main/resources/application.ymlbak @@ -19,4 +19,10 @@ jlog: server: - port: 8081 \ No newline at end of file + port: 8081 + + server.port=8083 + + #apollo.meta=http://127.0.0.1:8080 + #apollo.config-service=http://127.0.0.1:8080 + #app.id=order \ No newline at end of file diff --git a/clientdemo/src/main/resources/jLog.properties b/clientdemo/src/main/resources/jLog.properties new file mode 100644 index 0000000..3dd4dd0 --- /dev/null +++ b/clientdemo/src/main/resources/jLog.properties @@ -0,0 +1 @@ +reqTags=["AAA","CCC"] \ No newline at end of file diff --git a/config/config-core/pom.xml b/config/config-core/pom.xml index 16231fa..ca570b1 100644 --- a/config/config-core/pom.xml +++ b/config/config-core/pom.xml @@ -16,6 +16,11 @@ common 1.4-SNAPSHOT + + org.yaml + snakeyaml + 1.28 + diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java index 301b35a..4850f81 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -21,6 +21,10 @@ public class ConfigChangeEvent { } + /* public ConfigChangeEvent(String key){ + this(key, DEFAULT_NAMESPACE, null, null, ConfigChangeType.MODIFY); + }*/ + public ConfigChangeEvent(String key, String newValue) { this(key, DEFAULT_NAMESPACE, null, newValue, ConfigChangeType.MODIFY); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java index 83994aa..8439590 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -44,7 +44,7 @@ public interface ConfigChangeListener { * @param event 事件 */ default void onProcessEvent(ConfigChangeEvent event) { - LOGGER.info("通用监听器收到了变更事件, event={}", event.toString()); + LOGGER.info("通用监听器配置处理器", event.toString()); getExecutorService().submit(() -> { beforeEvent(); try { @@ -57,7 +57,7 @@ public interface ConfigChangeListener { } default void onShutDown() { - getExecutorService().shutdownNow(); + getExecutorService().shutdown(); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index c2829f6..d6d69d5 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -61,24 +61,24 @@ public interface Configurator { /** * 添加监听器 - * @param key key + * @param node 文件 */ - void addConfigListener(String key); + void addConfigListener(String node); /** * 移除监听器 - * @param key key + * @param node 节点 file or dir */ - void removeConfigListener(String key); + void removeConfigListener(String node); /** * 获取对应监听器 - * @param key key + * @param node key * @return Listeners */ - ConfigChangeListener getConfigListeners(String key); + ConfigChangeListener getConfigListeners(String node); /** * 获取配置器类型 diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java index ca5b759..18d72bc 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -75,7 +75,6 @@ public class ConfiguratorFactory { * 配置中心的配置器，如果没有就用文件配置器 */ ServiceLoader builders = ServiceLoader.load(ConfiguratorProvider.class); - LOGGER.info("ServiceLoader获取到的实现类:{}", builders.toString()); for (ConfiguratorProvider provider : builders) { LOGGER.info("第一个实现类类型为:{}", provider.build().getType()); return provider.build(); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index fe5b330..3df0341 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -1,14 +1,22 @@ package com.jd.platform.jlog.core; +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.common.utils.StringUtil; import io.netty.util.concurrent.DefaultThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; import java.io.*; import java.net.URL; +import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + + +import static com.jd.platform.jlog.core.FileConstant.*; /** @@ -21,57 +29,43 @@ public class FileConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class); - /** - * 监听重读配置文件间隔 单位ms - */ - private static final long LISTENER_CONFIG_INTERVAL = 5000; - - private static final String CONFIG_FILE_PROPERTIES = "/application.properties"; - - private static final String CONFIG_FILE_YML = "/application.yml"; - - private static final String JLOG_CONFIG_FILE = "/jLog.properties"; - - /** - * 配置文件集合 - */ - private static final String[] FILE_ARRAY = { CONFIG_FILE_PROPERTIES, CONFIG_FILE_YML, JLOG_CONFIG_FILE }; - - private final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(8); - - private final Map listenedConfigMap = new HashMap<>(8); + private static Properties PROPERTIES = new Properties(); - private final FileListener fileListener = new FileListener(); + private static volatile FileListener FILELISTENER = null; - private final Properties properties = new Properties(); + private static final Set LISTENED_FILES = new CopyOnWriteArraySet<>(); - private volatile long lastModify = 0L; + private final Map FILE_MODIFY_MAP = new ConcurrentHashMap<>(); - public FileConfigurator() throws IOException { + public FileConfigurator() throws IOException{ - String env = System.getenv("env"); + String env = System.getenv(ENV); - for (String file : FILE_ARRAY) { - - file = StringUtil.isEmpty(env) ? file : file + "_" + env; - - try (InputStream fis = this.getClass().getResourceAsStream(file)) { - if(fis != null){ - properties.load(fis); - LOGGER.info("{}配置文件配置:{}", file, properties.toString()); - }else{ - LOGGER.warn("{}配置文件为空",file); + for (String file : CONFIG_FILES) { + String fileName = StringUtil.isEmpty(env) ? file : file + "_" + env; + URL url = this.getClass().getResource(file); + if(url != null){ + Properties props = new Properties(); + try (InputStream is = url.openStream()) { + if (fileName.contains(YML)) { + props.putAll(new Yaml().loadAs(is, Map.class)); + } else { + props.load(is); + } + FILE_MODIFY_MAP.put(fileName, new FileWrapper(new File(url.getFile()).lastModified(), props)); + PROPERTIES.putAll(props); + LOGGER.info("{}配置文件配置:{}", file, props.toString()); } } } - LOGGER.info("合并后的配置:{}",properties.toString()); + LOGGER.info("合并后的配置:{}",PROPERTIES.toString()); } @Override public String getConfig(String key) { - return properties.getProperty(key); + return PROPERTIES.getProperty(key); } @@ -82,17 +76,16 @@ public class FileConfigurator implements Configurator { return value; } - if(properties.size() == 0){ + if(PROPERTIES.size() == 0){ // warn return null; } - return properties.getProperty(key); + return PROPERTIES.getProperty(key); } @Override - public boolean putConfig(String key, String content) { - return putConfig(key,content,2000L); - } + public boolean putConfig(String key, String content) { return false; } + @Override public boolean putConfig(String key, String content, long timeoutMills) { @@ -107,37 +100,60 @@ public class FileConfigurator implements Configurator { @Override - public void addConfigListener(String key) { - if (StringUtil.isBlank(key)) { - return; - } - FileListener fileListener = new FileListener(); - CONFIG_LISTENER_MAP.put(key, fileListener); - listenedConfigMap.put(key, ConfiguratorFactory.getInstance().getConfig(key,1000L)); - fileListener.addListener(key, fileListener); + public boolean removeConfig(String key) { + return false; } + @Override - public boolean removeConfig(String key) { - return false; + public void addConfigListener(String node) { + if (StringUtil.isBlank(node)) { + return; + } + String env = System.getenv(ENV); + + for (String file : CONFIG_FILES) { + file = StringUtil.isEmpty(env) ? file : file + "_" + env; + if(node.equals(file)){ + LISTENED_FILES.add(node); + if(FILELISTENER == null){ + synchronized (FileConfigurator.class){ + FILELISTENER = new FileListener(); + FILELISTENER.addListener(); + } + } + } + } } + @Override - public void removeConfigListener(String key) { - ConfigChangeListener configListener = getConfigListeners(key); - CONFIG_LISTENER_MAP.remove(key); - listenedConfigMap.remove(key); - configListener.onShutDown(); + public void removeConfigListener(String node) { + + LISTENED_FILES.remove(node); + if(LISTENED_FILES.isEmpty() && FILELISTENER != null){ + LOGGER.info("没有要监听的key了 关闭线程池"); + FILELISTENER.onShutDown(); + try { + if(!FILELISTENER.executor.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ + // 超时的时候向线程池中所有的线程发出中断(interrupted)。 + FILELISTENER.executor.shutdownNow(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + FILELISTENER = null; + } } @Override public ConfigChangeListener getConfigListeners(String key) { - return CONFIG_LISTENER_MAP.get(key); + return FILELISTENER; } @@ -147,6 +163,8 @@ public class FileConfigurator implements Configurator { return "file"; } + + @Override public Map getConfigByPrefix(String prefix) { return null; @@ -156,48 +174,29 @@ public class FileConfigurator implements Configurator { class FileListener implements ConfigChangeListener { - private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("fileListener", 1)); - FileListener() {} + FileListener() { } - public synchronized void addListener(String key, ConfigChangeListener listener) { - if (CONFIG_LISTENER_MAP.isEmpty()) { - fileListener.onProcessEvent(new ConfigChangeEvent()); - } - - CONFIG_LISTENER_MAP.put(key, listener); + public synchronized void addListener() { + FILELISTENER.onProcessEvent(new ConfigChangeEvent()); } + @Override public void onChangeEvent(ConfigChangeEvent event) { - while (true) { - for (String key : CONFIG_LISTENER_MAP.keySet()) { - try { - checkAndConfigure(); - String currentConfig = properties.getProperty(key); - if (currentConfig != null) { - String oldConfig = listenedConfigMap.get(key); - if (currentConfig.equals(oldConfig)) { - listenedConfigMap.put(key, currentConfig); - event.setKey(key).setNewValue(currentConfig).setOldValue(oldConfig); - CONFIG_LISTENER_MAP.get(key).onChangeEvent(event); - } - } - } catch (Exception e) { - LOGGER.error("fileListener execute error, key :{}", key, e); - } - } + for(; !getExecutorService().isShutdown() && !getExecutorService().isShutdown(); checkAndConfigure()) { try { Thread.sleep(LISTENER_CONFIG_INTERVAL); - } catch (InterruptedException e) { - LOGGER.error("fileListener thread sleep error:{}", e.getMessage()); + } catch (InterruptedException i) { + ; } } } + @Override public ExecutorService getExecutorService() { return executor; @@ -205,44 +204,102 @@ public class FileConfigurator implements Configurator { } + + /** * 检测文件最后修改时间 和重载文件 */ - private void checkAndConfigure() { - - File file; - FileInputStream inputStream = null; - - try { - file = new File(JLOG_CONFIG_FILE); + private void checkAndConfigure(){ + AtomicBoolean change = new AtomicBoolean(false); + Map newModifyMap = checkAndReload(); + FILE_MODIFY_MAP.forEach((k, v)->{ + FileWrapper newFile = newModifyMap.get(k); + if(newFile != null && newFile.change){ + boolean same = CollectionUtil.equals(newFile.props, v.props); + if(!same){ + LOGGER.warn("文件配置变更:{}", k, JSON.toJSONString(newFile.props)); + change.set(true); + v.props = newFile.props; + } + } + }); + if(change.get()){ + LOGGER.info("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); + PROPERTIES.clear(); + FILE_MODIFY_MAP.forEach((k,v)->{ + PROPERTIES.putAll(v.props); + }); + LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); + } + } - long l = file.lastModified(); - if(l <= this.lastModify){ - return; + /** + * 检测文件最后修改时间 和重载文件 + */ + private Map checkAndReload() { + + Map fileWrapperMap = new ConcurrentHashMap<>(3); + System.out.println("定时任务checkAndReload: "+JSON.toJSONString(LISTENED_FILES)); + for (String fileName : LISTENED_FILES) { + URL url = this.getClass().getResource(fileName); + if(url == null){ + continue; } - this.lastModify = l; - inputStream = new FileInputStream(file); - this.properties.clear(); - this.properties.load(inputStream); - inputStream.close(); - } catch (Exception e) { - if (e instanceof InterruptedIOException) { - Thread.currentThread().interrupt(); + File file = new File(url.getFile()); + long curLastMod = file.lastModified(); + FileWrapper wrapper = FILE_MODIFY_MAP.get(fileName); + Long cacheLastMod = wrapper.lastModify; + if(curLastMod <= cacheLastMod){ + continue; } - } finally { - if (inputStream != null) { - try { - inputStream.close(); - } catch (InterruptedIOException oe) { - Thread.currentThread().interrupt(); - } catch (Throwable ignored) { - + LOGGER.debug("定时扫描fileName:{} 更新了 最新时间:{} 缓存时间:{} threadName ={}",fileName,longToDate(curLastMod), longToDate(cacheLastMod),Thread.currentThread().getName()); + + Properties props = new Properties(); + try (InputStream is = url.openStream()) { + if (fileName.contains(YML)) { + props.putAll(new Yaml().loadAs(is, Map.class)); + } else { + props.load(is); } + fileWrapperMap.put(fileName, new FileWrapper(curLastMod, true, props)); + LOGGER.debug("定时任务：{}配置文件配置:{}", fileName, JSON.toJSONString(props)); + } catch (Exception e) { + LOGGER.error("定时任务load失败：{}配置文件配置重载失败", fileName, e); } } + return fileWrapperMap; + } + + + + public static String longToDate(long lo) { + + Date date = new Date(lo); + + SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + return sd.format(date); } + static class FileWrapper{ + + private long lastModify; + + private boolean change; + private Properties props; + + FileWrapper(long lastModify, Properties props) { + this.lastModify = lastModify; + this.change = false; + this.props = props; + } + FileWrapper(long lastModify, boolean change, Properties props) { + this.lastModify = lastModify; + this.change = change; + this.props = props; + } + } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java new file mode 100644 index 0000000..4ce50fc --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java @@ -0,0 +1,43 @@ +package com.jd.platform.jlog.core; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileConstant.java + * @Description TODO + * @createTime 2022年02月26日 10:19:00 + */ +public class FileConstant { + + /** + * 监听重读配置文件间隔 单位ms + */ + static final long LISTENER_CONFIG_INTERVAL = 10000; + + static final String CONFIG_FILE_PROPERTIES = "/application.properties"; + + static final String CONFIG_FILE_YML = "/application.yml"; + + static final String JLOG_CONFIG_FILE = "/jLog.properties"; + + static final String ENV = "env"; + + static final long AWAIT_TIME = 3 * 1000; + + static final String YML = "yml"; + + + /** + * 配置文件集合 + */ + public static final Set CONFIG_FILES = new HashSet() { + { + add(CONFIG_FILE_PROPERTIES); + add(CONFIG_FILE_YML); + add(JLOG_CONFIG_FILE); + } + }; +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index 882794e..5506f95 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -31,6 +31,7 @@ public class EtcdListener implements ConfigChangeListener { public EtcdListener(String key) { this.key = key; + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(key)).start(); System.out.println("构造器EtcdListener"); -- Gitee From 2df46b35349cc2f85a1807bd78c7a09a4228cce0 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Wed, 2 Mar 2022 16:19:00 +0800 Subject: [PATCH 13/22] =?UTF-8?q?=E7=9B=91=E5=90=AC=E5=99=A8=E4=B8=8D?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=96=E9=83=A8=E6=B7=BB=E5=8A=A0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 8 +- .../jlog/client/TracerClientStarter.java | 1 - .../jd/platform/jlog/client/task/Monitor.java | 13 +- clientdemo/pom.xml | 52 ++++-- clientdemo/src/main/resources/application.yml | 12 +- .../com/jd/platform/jlog/test/Common.java | 66 ++++++++ .../jlog/test/FileConfiguratorTest.java | 77 +++++++++ .../jlog/test/ZKConfiguratorTest.java | 91 ++++++++++ .../jlog/common/utils/CollectionUtil.java | 61 +++++++ .../jlog/common/utils/ConfigUtil.java | 27 +++ .../jlog/common/utils/FastJsonUtils.java | 2 +- .../config/apollo/ApolloConfigurator.java | 3 +- .../platform/jlog/core/ConfigChangeEvent.java | 8 +- .../jlog/core/ConfigChangeListener.java | 4 + .../jd/platform/jlog/core/Configurator.java | 24 ++- .../jlog/core/ConfiguratorFactory.java | 37 +--- .../core/{FileConstant.java => Constant.java} | 10 +- .../platform/jlog/core/FileConfigurator.java | 82 ++++----- .../platform/jlog/etcd/EtcdConfigurator.java | 67 ++++---- .../jd/platform/jlog/etcd/EtcdListener.java | 8 +- .../jlog/nacos/NacosConfigurator.java | 158 ++++++++---------- .../jd/platform/jlog/nacos/NacosListener.java | 56 +++---- .../jd/platform/jlog/zk/ZkConfigurator.java | 146 ++++++++-------- .../com/jd/platform/jlog/zk/ZkConstant.java | 3 +- .../com/jd/platform/jlog/zk/ZkListener.java | 34 ++-- 25 files changed, 656 insertions(+), 394 deletions(-) create mode 100644 clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java create mode 100644 clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java create mode 100644 clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java rename config/config-core/src/main/java/com/jd/platform/jlog/core/{FileConstant.java => Constant.java} (83%) diff --git a/client/pom.xml b/client/pom.xml index 9486691..26c0923 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,11 +18,11 @@ 1.4-SNAPSHOT - + - + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index 68ca85d..d252b50 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -7,7 +7,6 @@ import com.jd.platform.jlog.client.udp.HttpSender; import com.jd.platform.jlog.client.udp.UdpClient; import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.tag.TagConfig; -import com.jd.platform.jlog.common.tag.TagHandler; import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; diff --git a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java index df03264..92c4307 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java +++ b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java @@ -59,25 +59,22 @@ public class Monitor { private void fetch() throws Exception { Configurator configurator = ConfiguratorFactory.getInstance(); //获取所有worker的ip - Map keyVal = null; + List addresses = new ArrayList(); try { //如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有 if (Context.MDC != null) { String mdc = parseMdc(Context.MDC); - keyVal = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); + addresses = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); } - if (keyVal == null || keyVal.size() == 0) { - keyVal = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME); + if (addresses == null || addresses.size() == 0) { + addresses = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME); } //全是空，给个警告 - if (keyVal == null || keyVal.size() == 0) { + if (addresses == null || addresses.size() == 0) { LOGGER.warn("very important warn !!! workers ip info is null!!!"); return; } - - List addresses = new ArrayList<>(); - keyVal.forEach( (k ,v)-> addresses.add(v)); //将对应的worker保存下来 WorkerInfoHolder.mergeAndConnectNew(addresses); } catch (Exception ex) { diff --git a/clientdemo/pom.xml b/clientdemo/pom.xml index 24464dc..09a2c83 100644 --- a/clientdemo/pom.xml +++ b/clientdemo/pom.xml @@ -40,30 +40,54 @@ spring-boot-configuration-processor true - - + + + diff --git a/clientdemo/src/main/resources/application.yml b/clientdemo/src/main/resources/application.yml index 2ec88a2..faa4d69 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/clientdemo/src/main/resources/application.yml @@ -1,5 +1,11 @@ -serverAddr: 127.0.0.1:111 -a: 3333 +testKey: 111 +serverAddr: 101.42.242.201:2181 +server: + port: 8085 +workers: + - 111 + - 222 + - 333 tag-config: reqTags: @@ -11,6 +17,4 @@ tag-config: regex: "" delimiter: "|" join: = -server: - port: 8085 diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java b/clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java new file mode 100644 index 0000000..37242b1 --- /dev/null +++ b/clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java @@ -0,0 +1,66 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.core.Configurator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.List; +import java.util.Random; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Common.java + * @Description TODO + * @createTime 2022年03月01日 07:36:00 + */ +public class Common { + + private final static Logger LOGGER = LoggerFactory.getLogger(Common.class); + + + public static void getTest(Configurator configurator){ + + LOGGER.info("配置器类型：{}", configurator.getType()); + String addr = configurator.getConfig("serverAddr"); + LOGGER.info("配置器get addr：{}", addr); + String reqTags = configurator.getConfig("reqTags"); + LOGGER.info("配置器get reqTags：{}", reqTags); + List workers = configurator.getConfigByPrefix("workers"); + LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); + } + + + public static void modifyFile(String path)throws Exception{ + + String temp; + File file = new File(path); + FileInputStream fis = new FileInputStream(file); + InputStreamReader isr = new InputStreamReader(fis); + BufferedReader br = new BufferedReader(isr); + StringBuffer buf = new StringBuffer(); + + int id = new Random().nextInt(1000); + int num = 0; + // 保存该行前面的内容 + while ((temp = br.readLine()) != null) { + if(num == 0){ + buf = buf.append("testKey: ").append(id); + }else{ + buf = buf.append(temp); + } + num++; + buf = buf.append(System.getProperty("line.separator")); + } + + br.close(); + FileOutputStream fos = new FileOutputStream(file); + PrintWriter pw = new PrintWriter(fos); + pw.write(buf.toString().toCharArray()); + pw.flush(); + pw.close(); + } + +} diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java new file mode 100644 index 0000000..234c90b --- /dev/null +++ b/clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -0,0 +1,77 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; +import static com.jd.platform.jlog.test.Common.modifyFile; + + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileConfiguratorTest.java + * @Description TODO + * @createTime 2022年02月28日 19:45:00 + */ +@SpringBootTest(classes = ClientDemoApplication.class) +@RunWith(SpringRunner.class) +public class FileConfiguratorTest { + + + private final static Logger LOGGER = LoggerFactory.getLogger(FileConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + + + @Test + public void testAddConfigListener() throws Exception { + configurator.addConfigListener("/application.yml"); + String path = "/Users/didi/Desktop/jlog/clientdemo/target/classes/application.yml"; + Properties props = new Properties(); + FileInputStream fis = new FileInputStream(new File(path)); + if (path.contains("yml")) { + props.putAll(new Yaml().loadAs(fis, Map.class)); + } else { + props.load(fis); + } + LOGGER.info("读取文件：{}最新配置:{}", path, JSON.toJSONString(props)); + modifyFile(path); + LOGGER.info("修改文件完毕 准备触发监听器"); + Thread.sleep(1000); + LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getConfig("testKey")); + LOGGER.info("移除监听器"); + configurator.removeConfigListener("/application.yml"); + modifyFile(path); + LOGGER.info("修改文件完毕 准备验证监听器是否停止 最新testKey={}", configurator.getConfig("testKey")); + } + + + +} diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java b/clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java new file mode 100644 index 0000000..5850d22 --- /dev/null +++ b/clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java @@ -0,0 +1,91 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileInputStream; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; +import static com.jd.platform.jlog.test.Common.modifyFile; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ZKConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月01日 07:35:00 + */ +@SpringBootTest(classes = ClientDemoApplication.class) +@RunWith(SpringRunner.class) +public class ZKConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(ZKConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getConfigByPrefix("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.1.1.1"; + if(workers.contains(myIp)){ + // do nothing + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getConfigByPrefix("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getConfig("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + configurator.addConfigListener("/application.yml"); + LOGGER.info("添加监听器:随机数：{}", i1); + configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(1000); + String val2 = configurator.getConfig("testKey"); + LOGGER.info("修改后的的val:{}", val2); + LOGGER.info("移除监听器:随机数：{}",i2); + Thread.sleep(3000); + configurator.removeConfigListener("/application.yml"); + Thread.sleep(1000); + configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getConfig("testKey")); + } +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java index 426b219..5890083 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java @@ -1,8 +1,11 @@ package com.jd.platform.jlog.common.utils; +import com.alibaba.fastjson.JSON; + import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author tangbohu @@ -81,4 +84,62 @@ public class CollectionUtil { } } + + + + + public static Set diffKeys(Map m1, Map m2){ + + Set diff = new HashSet<>(1); + + for (Map.Entry kvEntry : m1.entrySet()) { + V val = m2.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.add(kvEntry.getKey().toString()); + } + } + + for (Map.Entry kvEntry : m2.entrySet()) { + V val = m1.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.add(kvEntry.getKey().toString()); + } + } + return diff; + } + + + + + public static HashMap diffMap(Map m1, Map m2){ + + HashMap diff = new HashMap<>(1); + + for (Map.Entry kvEntry : m1.entrySet()) { + V val = m2.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.put(kvEntry.getKey(), kvEntry.getValue()); + } + } + + for (Map.Entry kvEntry : m2.entrySet()) { + V val = m1.get(kvEntry.getKey()); + if(!kvEntry.getValue().equals(val)){ + diff.put(kvEntry.getKey(), kvEntry.getValue()); + } + } + return diff; + } + + + public static void main(String[] args) { + HashMap m1 = new HashMap<>(); + m1.put("t1",1); + m1.put("t2",2); + + HashMap m2 = new HashMap<>(); + m2.put("t2",2); + m2.put("t3",3); + + } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java index e7e1d67..e74de5e 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -1,6 +1,9 @@ package com.jd.platform.jlog.common.utils; import com.jd.platform.jlog.common.constant.Constant; + +import java.util.Enumeration; +import java.util.Properties; import java.util.concurrent.ThreadLocalRandom; /** @@ -31,6 +34,30 @@ public class ConfigUtil { return str; } + + + public static String formatConfigStr(Properties properties) { + + StringBuilder sb = new StringBuilder(); + Enumeration enumeration = properties.propertyNames(); + while (enumeration.hasMoreElements()) { + String key = (String) enumeration.nextElement(); + Object property = properties.get(key); + if(property != null){ + property = String.valueOf(property); + } + sb.append(key).append("=").append(property).append("\n"); + } + return sb.toString(); + } + + + + public static byte[] formatConfigByte(Properties properties) { + return formatConfigStr(properties).getBytes(); + } + + /* public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception { diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java index 2b2ebe6..976e1ef 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java @@ -72,7 +72,7 @@ public class FastJsonUtils { /** * json字符串转化为map */ - public static Map stringToCollect(String s) { + public static Map toMap(String s) { return (Map) JSONObject.parseObject(s); } diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index 46b3e2d..6c5a889 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -1,5 +1,6 @@ package com.jd.platform.jlog.config.apollo; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.*; @@ -141,7 +142,7 @@ public class ApolloConfigurator implements Configurator { @Override - public Map getConfigByPrefix(String prefix) { + public List getConfigByPrefix(String prefix) { return null; } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java index 4850f81..7953690 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -21,14 +21,14 @@ public class ConfigChangeEvent { } - /* public ConfigChangeEvent(String key){ - this(key, DEFAULT_NAMESPACE, null, null, ConfigChangeType.MODIFY); - }*/ - public ConfigChangeEvent(String key, String newValue) { this(key, DEFAULT_NAMESPACE, null, newValue, ConfigChangeType.MODIFY); } + public ConfigChangeEvent(String key, String oldValue, String newValue){ + this(key, DEFAULT_NAMESPACE, oldValue, newValue, ConfigChangeType.MODIFY); + } + public ConfigChangeEvent(String key, String namespace, String oldValue, String newValue, ConfigChangeType type) { this.key = key; diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java index 8439590..5c790ee 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -36,6 +36,10 @@ public interface ConfigChangeListener { Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), Executors.defaultThreadFactory()); + /** + * 事件具体处理 + * @param event event + */ void onChangeEvent(ConfigChangeEvent event); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index d6d69d5..36df609 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -1,5 +1,6 @@ package com.jd.platform.jlog.core; +import java.util.List; import java.util.Map; /** @@ -39,6 +40,7 @@ public interface Configurator { * @param key key * @return content val * @param timeoutMills timeoutMills + * @param content content */ boolean putConfig(String key, String content, long timeoutMills); @@ -59,6 +61,14 @@ public interface Configurator { boolean removeConfig(String key, long timeoutMills); + /** + * 根据前缀/父级路径获取子节点 + * @param prefix path + * @return List + */ + List getConfigByPrefix(String prefix); + + /** * 添加监听器 * @param node 文件 @@ -73,23 +83,11 @@ public interface Configurator { void removeConfigListener(String node); - /** - * 获取对应监听器 - * @param node key - * @return Listeners - */ - ConfigChangeListener getConfigListeners(String node); - /** * 获取配置器类型 * @return string example:apollo */ String getType(); - /** - * 根据前缀/父级路径获取子节点 - * @param prefix path - * @return List - */ - Map getConfigByPrefix(String prefix); + } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java index 18d72bc..92e7795 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -1,18 +1,3 @@ -/* - * Copyright 1999-2019 Seata.io Group. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.jd.platform.jlog.core; @@ -63,31 +48,25 @@ public class ConfiguratorFactory { private static Configurator buildConfiguration() { try { - /** - * 配置文件的配置器 + /* + 配置文件的配置器 */ base = new FileConfigurator(); } catch (IOException e) { - e.printStackTrace(); + LOGGER.info("文件配置器构建失败", e); + throw new RuntimeException("build file buildConfiguration fail", e); } - /** - * 配置中心的配置器，如果没有就用文件配置器 + /* + 配置中心的配置器，如果没有就用文件配置器 */ ServiceLoader builders = ServiceLoader.load(ConfiguratorProvider.class); + //noinspection LoopStatementThatDoesntLoop for (ConfiguratorProvider provider : builders) { - LOGGER.info("第一个实现类类型为:{}", provider.build().getType()); + LOGGER.info("配置中心的配置器获取成功, 类型为:{}", provider.build().getType()); return provider.build(); } return base; } - /** - * 重载 - * @throws Exception - */ - protected static void reload() throws Exception { - instance = null; - getInstance(); - } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java similarity index 83% rename from config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java rename to config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java index 4ce50fc..f40ea23 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConstant.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java @@ -6,11 +6,11 @@ import java.util.Set; /** * @author tangbohu * @version 1.0.0 - * @ClassName FileConstant.java + * @ClassName Constant.java * @Description TODO * @createTime 2022年02月26日 10:19:00 */ -public class FileConstant { +public class Constant { /** * 监听重读配置文件间隔 单位ms @@ -23,12 +23,18 @@ public class FileConstant { static final String JLOG_CONFIG_FILE = "/jLog.properties"; + static final String ENV = "env"; static final long AWAIT_TIME = 3 * 1000; static final String YML = "yml"; + public static final long DEFAULT_TIMEOUT = 2 * 1000; + + public static final String SERVER_ADDR_KEY = "serverAddr"; + + /** * 配置文件集合 diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index 3df0341..20730b1 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.common.utils.StringUtil; import io.netty.util.concurrent.DefaultThreadFactory; import org.slf4j.Logger; @@ -10,13 +11,11 @@ import org.yaml.snakeyaml.Yaml; import java.io.*; import java.net.URL; -import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; - -import static com.jd.platform.jlog.core.FileConstant.*; +import static com.jd.platform.jlog.core.Constant.*; /** @@ -38,7 +37,7 @@ public class FileConfigurator implements Configurator { private final Map FILE_MODIFY_MAP = new ConcurrentHashMap<>(); - public FileConfigurator() throws IOException{ + FileConfigurator() throws IOException{ String env = System.getenv(ENV); @@ -46,8 +45,8 @@ public class FileConfigurator implements Configurator { String fileName = StringUtil.isEmpty(env) ? file : file + "_" + env; URL url = this.getClass().getResource(file); if(url != null){ - Properties props = new Properties(); try (InputStream is = url.openStream()) { + Properties props = new Properties(); if (fileName.contains(YML)) { props.putAll(new Yaml().loadAs(is, Map.class)); } else { @@ -65,24 +64,20 @@ public class FileConfigurator implements Configurator { @Override public String getConfig(String key) { - return PROPERTIES.getProperty(key); + Object val = PROPERTIES.get(key); + if(val != null){ + return String.valueOf(val); + } + return null; } @Override public String getConfig(String key, long timeoutMills) { - String value = System.getProperty(key); - if (value != null) { - return value; - } - - if(PROPERTIES.size() == 0){ - // warn - return null; - } return PROPERTIES.getProperty(key); } + @Override public boolean putConfig(String key, String content) { return false; } @@ -103,8 +98,7 @@ public class FileConfigurator implements Configurator { public boolean removeConfig(String key) { return false; } - - + @Override public void addConfigListener(String node) { @@ -152,25 +146,24 @@ public class FileConfigurator implements Configurator { @Override - public ConfigChangeListener getConfigListeners(String key) { - return FILELISTENER; - } - + public List getConfigByPrefix(String prefix) { + Object val = PROPERTIES.get(prefix); - @Override - public String getType() { - return "file"; + if(val != null){ + return FastJsonUtils.toList(String.valueOf(val), String.class); + } + return null; } - @Override - public Map getConfigByPrefix(String prefix) { - return null; + public String getType() { + return "file"; } + class FileListener implements ConfigChangeListener { @@ -180,7 +173,7 @@ public class FileConfigurator implements Configurator { FileListener() { } - public synchronized void addListener() { + synchronized void addListener() { FILELISTENER.onProcessEvent(new ConfigChangeEvent()); } @@ -190,8 +183,7 @@ public class FileConfigurator implements Configurator { for(; !getExecutorService().isShutdown() && !getExecutorService().isShutdown(); checkAndConfigure()) { try { Thread.sleep(LISTENER_CONFIG_INTERVAL); - } catch (InterruptedException i) { - ; + } catch (InterruptedException ignored) { } } } @@ -215,20 +207,22 @@ public class FileConfigurator implements Configurator { FILE_MODIFY_MAP.forEach((k, v)->{ FileWrapper newFile = newModifyMap.get(k); if(newFile != null && newFile.change){ - boolean same = CollectionUtil.equals(newFile.props, v.props); - if(!same){ - LOGGER.warn("文件配置变更:{}", k, JSON.toJSONString(newFile.props)); + + Set diffKeys = CollectionUtil.diffKeys(newFile.props, v.props); + if(!diffKeys.isEmpty()){ change.set(true); v.props = newFile.props; + + for (String diffKey : diffKeys) { + LOGGER.warn("文件 {} 配置变更 key={}变更事件:{}", k, diffKey, new ConfigChangeEvent(diffKey, v.props.get(diffKey).toString(), newFile.props.get(diffKey).toString())); + } } } }); if(change.get()){ LOGGER.info("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); PROPERTIES.clear(); - FILE_MODIFY_MAP.forEach((k,v)->{ - PROPERTIES.putAll(v.props); - }); + FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props)); LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); } } @@ -240,7 +234,6 @@ public class FileConfigurator implements Configurator { private Map checkAndReload() { Map fileWrapperMap = new ConcurrentHashMap<>(3); - System.out.println("定时任务checkAndReload: "+JSON.toJSONString(LISTENED_FILES)); for (String fileName : LISTENED_FILES) { URL url = this.getClass().getResource(fileName); if(url == null){ @@ -253,8 +246,6 @@ public class FileConfigurator implements Configurator { if(curLastMod <= cacheLastMod){ continue; } - LOGGER.debug("定时扫描fileName:{} 更新了 最新时间:{} 缓存时间:{} threadName ={}",fileName,longToDate(curLastMod), longToDate(cacheLastMod),Thread.currentThread().getName()); - Properties props = new Properties(); try (InputStream is = url.openStream()) { if (fileName.contains(YML)) { @@ -263,7 +254,7 @@ public class FileConfigurator implements Configurator { props.load(is); } fileWrapperMap.put(fileName, new FileWrapper(curLastMod, true, props)); - LOGGER.debug("定时任务：{}配置文件配置:{}", fileName, JSON.toJSONString(props)); + LOGGER.info("定时任务监测到文件更新：{}最新配置:{}", fileName, JSON.toJSONString(props)); } catch (Exception e) { LOGGER.error("定时任务load失败：{}配置文件配置重载失败", fileName, e); } @@ -272,17 +263,6 @@ public class FileConfigurator implements Configurator { } - - public static String longToDate(long lo) { - - Date date = new Date(lo); - - SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - - return sd.format(date); - } - - static class FileWrapper{ private long lastModify; diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index deb48d0..bf55fcb 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -16,6 +16,8 @@ import com.jd.platform.jlog.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; + /** * @author tangbohu * @version 1.0.0 @@ -33,22 +35,24 @@ public class EtcdConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; - private ConcurrentMap configListenerMap = new ConcurrentHashMap<>(8); - - private static final String SERVER_ADDR_KEY = "serverAddr"; - private static final String ROOT = "/jLog"; private static final String PROPERTIES_PATH = "/properties"; private static Properties PROPERTIES = new Properties(); + private volatile static EtcdListener LISTENER = null; private EtcdConfigurator() { LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); client = EtcdClient.forEndpoints(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,2000L)).withPlainText().build(); - String val = getConfig(ROOT + PROPERTIES_PATH); + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(ROOT + PROPERTIES_PATH)).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return; + } + String val = keyValues.get(0).getValue().toStringUtf8(); if(StringUtil.isNotBlank(val)){ PROPERTIES.putAll((Map)JSON.parse(val)); } @@ -69,22 +73,16 @@ public class EtcdConfigurator implements Configurator { @Override public String getConfig(String key) { - RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(key)).sync(); - List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { - return null; + Object val = PROPERTIES.get(key); + if(val != null){ + return String.valueOf(val); } - return keyValues.get(0).getValue().toStringUtf8(); + return null; } @Override public String getConfig(String key, long timeoutMills) { - RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(key)).timeout(timeoutMills).sync(); - List keyValues = rangeResponse.getKvsList(); - if (CollectionUtil.isEmpty(keyValues)) { - return null; - } - return keyValues.get(0).getValue().toStringUtf8(); + return getConfig(key); } @Override @@ -113,28 +111,19 @@ public class EtcdConfigurator implements Configurator { } @Override - public void addConfigListener(String key) { - System.out.println("添加etcd监听器"+key); - EtcdListener etcdListener = new EtcdListener(key); - configListenerMap.put(key, etcdListener); - etcdListener.onProcessEvent(new ConfigChangeEvent()); - } - - @Override - public void removeConfigListener(String key) { - System.out.println("移除etcd监听器"+key); - ConfigChangeListener configListener = getConfigListeners(key); - configListenerMap.remove(key); - configListener.onShutDown(); + public void addConfigListener(String node) { + System.out.println("添加etcd监听器"+node); + LISTENER = new EtcdListener(node); + LISTENER.onProcessEvent(new ConfigChangeEvent()); } - @Override - public ConfigChangeListener getConfigListeners(String key) { - return configListenerMap.get(key); + public void removeConfigListener(String node) { + System.out.println("移除etcd监听器"+node); + LISTENER.onShutDown(); + LISTENER = null; } - @Override public String getType() { return "etcd"; @@ -142,14 +131,14 @@ public class EtcdConfigurator implements Configurator { @Override - public Map getConfigByPrefix(String prefix) { + public List getConfigByPrefix(String prefix) { RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(prefix)).asPrefix().sync(); - List list = rangeResponse.getKvsList(); - Map map = new HashMap<>(list.size()); - for (KeyValue kv : list) { - map.put(kv.getKey().toStringUtf8(), kv.getValue().toStringUtf8()); + List keyValues = rangeResponse.getKvsList(); + List list = new ArrayList<>(); + for (KeyValue kv : keyValues) { + list.add(kv.getValue().toStringUtf8()); } - return map; + return list; } } diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index 5506f95..6524c83 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -23,16 +23,14 @@ import java.util.concurrent.TimeUnit; * @createTime 2022年02月21日 23:34:00 */ public class EtcdListener implements ConfigChangeListener { - private final String key; private KvClient.WatchIterator iterator; private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("etcdListener", 1)); - public EtcdListener(String key) { - this.key = key; + public EtcdListener(String node) { - iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(key)).start(); + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(node)).start(); System.out.println("构造器EtcdListener"); getExecutorService().submit(() -> { @@ -43,7 +41,7 @@ public class EtcdListener implements ConfigChangeListener { ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; ConfigChangeEvent event = new ConfigChangeEvent(); - event.setKey(key).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + event.setKey(node).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); onChangeEvent(event); } }); diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java index 9740d7b..678d071 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -1,9 +1,6 @@ package com.jd.platform.jlog.nacos; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; +import java.io.*; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -15,20 +12,19 @@ import com.alibaba.nacos.api.config.ConfigService; import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.utils.StringUtil; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; +import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; import static com.jd.platform.jlog.nacos.NacosConstant.*; -/** - * The type Nacos configuration. - * - * @author slievrly - */ + public class NacosConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); @@ -41,9 +37,11 @@ public class NacosConfigurator implements Configurator { private static volatile ConfigService configService; - static final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(8); + static final ConcurrentMap KEY_DATAID_MAP = new ConcurrentHashMap<>(8); + + static volatile Properties PROPERTIES = new Properties(); - static volatile Properties props = new Properties(); + static NacosListener NACOSLISTENER = new NacosListener(); public static NacosConfigurator getInstance() { @@ -62,9 +60,18 @@ public class NacosConfigurator implements Configurator { if (configService == null) { try { configService = NacosFactory.createConfigService(getConfigProperties()); - LOGGER.info("实例化NacosConfigurator完成 result = {}",configService.getServerStatus()); - initConfig(); - } catch (NacosException e) { + String config = configService.getConfig(DEFAULT_DATA_ID, JLOG_GROUP, DEFAULT_TIMEOUT); + LOGGER.info("从NaCos获取配置进行初始化 config = {}", config); + if (StringUtils.isNotBlank(config)) { + PROPERTIES.load(new StringReader(config)); + LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(PROPERTIES)); + configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, NACOSLISTENER); + Enumeration e = PROPERTIES.propertyNames(); + while (e.hasMoreElements()) { + KEY_DATAID_MAP.put((String) e.nextElement(), DEFAULT_DATA_ID); + } + } + } catch (NacosException | IOException e) { throw new RuntimeException(e); } } @@ -73,9 +80,10 @@ public class NacosConfigurator implements Configurator { @Override public String getConfig(String key) { - return null; + return getConfig(key, DEFAULT_TIMEOUT); } + @Override public String getConfig(String key, long timeoutMills) { String value = System.getProperty(key); @@ -83,48 +91,73 @@ public class NacosConfigurator implements Configurator { return value; } - value = props.getProperty(key); + value = PROPERTIES.getProperty(key); if (null == value) { try { - value = configService.getConfig(key, JLOG_GROUP, timeoutMills); - } catch (NacosException exx) { - LOGGER.error(exx.getErrMsg()); + String dataId = KEY_DATAID_MAP.get(key); + if(StringUtil.isEmpty(dataId)){ + return null; + } + String config = configService.getConfig(dataId, JLOG_GROUP, timeoutMills); + if(StringUtil.isEmpty(config)){ + return null; + } + PROPERTIES.load(new StringReader(config)); + } catch (NacosException | IOException ex) { + LOGGER.error(ex.getMessage()); } } - - return value; + return PROPERTIES.getProperty(key); } + @Override public boolean putConfig(String key, String content) { - return false; + return putConfig(key, content, DEFAULT_TIMEOUT); } + @Override - public boolean putConfig(String dataId, String content, long timeoutMills) { + public boolean putConfig(String key, String content, long timeoutMills) { boolean result = false; + String dataId = KEY_DATAID_MAP.get(key); + if(StringUtil.isEmpty(dataId)){ + return false; + } try { - if (!props.isEmpty()) { - props.setProperty(dataId, content); - result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, getConfigStr()); + if (!PROPERTIES.isEmpty()) { + PROPERTIES.setProperty(key, content); + result = configService.publishConfig(dataId, JLOG_GROUP, formatConfigStr(PROPERTIES)); } else { result = configService.publishConfig(dataId, JLOG_GROUP, content); } - } catch (NacosException exx) { - LOGGER.error(exx.getErrMsg()); + } catch (NacosException ex) { + LOGGER.error(ex.getErrMsg()); } return result; } @Override - public boolean removeConfig(String dataId, long timeoutMills) { + public boolean removeConfig(String key) { + return removeConfig(key, DEFAULT_TIMEOUT); + } + + + + @Override + public boolean removeConfig(String key, long timeoutMills) { + + String dataId = KEY_DATAID_MAP.get(key); + if(StringUtil.isEmpty(dataId)){ + return false; + } boolean result = false; try { - if (!props.isEmpty()) { - props.remove(dataId); - result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, getConfigStr()); + if (!PROPERTIES.isEmpty()) { + PROPERTIES.remove(key); + result = configService.publishConfig(dataId, JLOG_GROUP, formatConfigStr(PROPERTIES)); } else { result = configService.removeConfig(dataId, JLOG_GROUP); } @@ -135,19 +168,15 @@ public class NacosConfigurator implements Configurator { } - @Override - public boolean removeConfig(String key) { - return false; - } - @Override public void addConfigListener(String dataId) { + if(!DEFAULT_DATA_ID.equals(dataId)){ + throw new RuntimeException("no support"); + } LOGGER.info("nacos添加监听器开始 dataId:{}",dataId); - NacosListener nacosListener = new NacosListener(dataId); - CONFIG_LISTENER_MAP.put(dataId, nacosListener); + NacosListener nacosListener = new NacosListener(); LOGGER.info("## nacos添加监听器过程 {}",nacosListener.toString()); - try { configService.addListener(dataId, JLOG_GROUP, nacosListener); } catch (NacosException e) { @@ -159,21 +188,11 @@ public class NacosConfigurator implements Configurator { @Override public void removeConfigListener(String dataId) { - if (StringUtils.isBlank(dataId)) { - return; + if(!DEFAULT_DATA_ID.equals(dataId)){ + throw new RuntimeException("no support"); } LOGGER.info("nacos移除监听器开始"); - ConfigChangeListener changeListener = getConfigListeners(dataId); - NacosListener listener = (NacosListener) changeListener; - LOGGER.info("## nacos移除监听器过程 {}",listener.toString()); - - CONFIG_LISTENER_MAP.remove(dataId); - configService.removeListener(dataId, JLOG_GROUP, listener); - } - - @Override - public ConfigChangeListener getConfigListeners(String dataId) { - return CONFIG_LISTENER_MAP.get(dataId); + configService.removeListener(dataId, JLOG_GROUP, NACOSLISTENER); } @@ -188,37 +207,6 @@ public class NacosConfigurator implements Configurator { - private static String getConfigStr() { - StringBuilder sb = new StringBuilder(); - - Enumeration enumeration = props.propertyNames(); - while (enumeration.hasMoreElements()) { - String key = (String) enumeration.nextElement(); - String property = props.getProperty(key); - sb.append(key).append("=").append(property).append("\n"); - } - - return sb.toString(); - } - - private static void initConfig() { - try { - String config = configService.getConfig(DEFAULT_DATA_ID, JLOG_GROUP, 2000L); - LOGGER.info("从NaCos获取配置进行初始化 config = {}", config); - - if (StringUtils.isNotBlank(config)) { - try (Reader reader = new InputStreamReader(new ByteArrayInputStream(config.getBytes()), StandardCharsets.UTF_8)) { - props.load(reader); - } - LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(props)); - NacosListener nacosListener = new NacosListener(DEFAULT_DATA_ID); - configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, nacosListener); - } - } catch (NacosException | IOException e) { - LOGGER.error("init config properties error", e); - } - } - @Override public String getType() { return "nacos"; @@ -226,7 +214,7 @@ public class NacosConfigurator implements Configurator { @Override - public Map getConfigByPrefix(String prefix) { + public List getConfigByPrefix(String prefix) { return null; } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java index d4415a3..da7d2c7 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -1,6 +1,5 @@ package com.jd.platform.jlog.nacos; -import com.alibaba.fastjson.JSON; import com.alibaba.nacos.api.config.listener.AbstractSharedListener; import com.alibaba.nacos.api.naming.listener.Event; import com.alibaba.nacos.api.naming.listener.EventListener; @@ -13,55 +12,26 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.util.Properties; +import java.util.Set; -import static com.jd.platform.jlog.nacos.NacosConfigurator.CONFIG_LISTENER_MAP; -import static com.jd.platform.jlog.nacos.NacosConfigurator.props; +import static com.jd.platform.jlog.nacos.NacosConfigurator.*; import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; -import static com.jd.platform.jlog.nacos.NacosConstant.JLOG_GROUP; public class NacosListener extends AbstractSharedListener implements ConfigChangeListener, EventListener { - private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); + private static final Logger LOGGER = LoggerFactory.getLogger(NacosListener.class); - private final String dataId; - private final String group; - - - public NacosListener(String dataId) { - this.dataId = dataId; - this.group = JLOG_GROUP; - } + public NacosListener() {} @Override public void innerReceive(String dataId, String group, String configInfo) { LOGGER.info("configInfo:{}", configInfo); - System.out.println("自带监听器" + configInfo); - if (DEFAULT_DATA_ID.equals(dataId)) { - Properties properties = new Properties(); - if (StringUtils.isNotBlank(configInfo)) { - try { - properties.load(new StringReader(configInfo)); - } catch (IOException e) { - return; - } - } - LOGGER.info("CONFIG_LISTENER_MAP :{}",JSON.toJSONString(CONFIG_LISTENER_MAP)); - ConfigChangeListener listener = CONFIG_LISTENER_MAP.get(dataId); - boolean same = CollectionUtil.equals(properties, props); - if(!same){ - ConfigChangeEvent event = new ConfigChangeEvent() - .setKey(DEFAULT_DATA_ID) - .setOldValue(JSON.toJSONString(props)) - .setNewValue(JSON.toJSONString(properties)) - .setNamespace(group); - listener.onProcessEvent(event); - props = properties; - } + if(!DEFAULT_DATA_ID.equals(dataId)){ return; } @@ -75,6 +45,22 @@ public class NacosListener extends AbstractSharedListener implements ConfigChang public void onChangeEvent(ConfigChangeEvent event) { System.out.println("通用[配置]变更事件"); LOGGER.info("通用[配置]变更事件 event:{}",event); + + Properties props = new Properties(); + if (StringUtils.isNotBlank(event.getNewValue())) { + try { + props.load(new StringReader(event.getNewValue())); + } catch (IOException e) { + return; + } + } + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); + if(!diffKeys.isEmpty()){ + PROPERTIES = props; + for (String diffKey : diffKeys) { + LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", DEFAULT_DATA_ID, diffKey, new ConfigChangeEvent(diffKey, props.get(diffKey).toString(), PROPERTIES.get(diffKey).toString())); + } + } } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java index 11ff738..bea239a 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -1,27 +1,28 @@ package com.jd.platform.jlog.zk; import java.io.ByteArrayInputStream; -import java.util.Enumeration; -import java.util.Map; +import java.util.List; import java.util.Properties; -import java.util.concurrent.*; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.common.utils.StringUtil; import com.jd.platform.jlog.core.*; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.jd.platform.jlog.zk.ZkConstant.DEFAULT_CONFIG_PATH; -import static com.jd.platform.jlog.zk.ZkConstant.NAMESPACE; -import static com.jd.platform.jlog.zk.ZkConstant.SERVER_ADDR_KEY; +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte; +import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; +import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; +import static com.jd.platform.jlog.zk.ZkConstant.*; /** - * @author didi + * @author tangbohu */ public class ZkConfigurator implements Configurator { @@ -31,33 +32,47 @@ public class ZkConfigurator implements Configurator { static volatile CuratorFramework zkClient; - static final ConcurrentMap CONFIG_LISTENERS_MAP = new ConcurrentHashMap<>(8); - static volatile Properties pros = new Properties(); + static volatile Properties PROPERTIES = new Properties(); + + private static volatile ZkListener ZKLISTENER = null; + public ZkConfigurator() throws Exception { + System.out.println("### SERVER_ADDR_KEY ===> "+FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); if (zkClient == null) { synchronized (ZkConfigurator.class) { zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getConfig(SERVER_ADDR_KEY)) // 连接超时时间 - .sessionTimeoutMs(2000) + .sessionTimeoutMs(6000) // 会话超时时间 - .connectionTimeoutMs(6000) + .connectionTimeoutMs(2000) .namespace(NAMESPACE) // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 .retryPolicy(new ExponentialBackoffRetry(1000, 3)) .build(); zkClient.start(); } + + if(zkClient.checkExists().forPath(DEFAULT_WORKER_PATH) == null){ + zkClient.create().withMode(CreateMode.EPHEMERAL).forPath(DEFAULT_WORKER_PATH); + } loadZkData(); - LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(pros)); + addConfigListener(DEFAULT_CONFIG_PATH); + LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(PROPERTIES)); } } @Override public String getConfig(String key) { - String value = pros.getProperty(key); + return getConfig(key, DEFAULT_TIMEOUT); + } + + + @Override + public String getConfig(String key, long timeoutMills) { + String value = PROPERTIES.getProperty(key); if (value != null) { return value; } @@ -67,96 +82,92 @@ public class ZkConfigurator implements Configurator { return value; } try { - return new String(zkClient.getData().forPath(DEFAULT_CONFIG_PATH)); + loadZkData(); } catch (Exception e) { - e.printStackTrace(); + return null; } - return null; + return PROPERTIES.getProperty(key); } - @Override - public String getConfig(String key, long timeoutMills) { - return getConfig(key); - } - @Override public boolean putConfig(String key, String content) { - if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ - return false; - } - pros.setProperty(key, content); - return true; + return putConfig(key, content, DEFAULT_TIMEOUT); } @Override public boolean putConfig(String key, String content, long timeoutMills) { - pros.setProperty(key, content); + if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ + return false; + } + PROPERTIES.setProperty(key, content); try { - zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigStr()); + zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigByte(PROPERTIES)); } catch (Exception e) { - e.printStackTrace(); + return false; } return true; } - @Override - public boolean removeConfig(String dataId, long timeoutMills) { - pros.remove(dataId); - try { - zkClient.delete().forPath(DEFAULT_CONFIG_PATH); - } catch (Exception e) { - e.printStackTrace(); - } - return true; - } @Override public boolean removeConfig(String key) { - return false; + return removeConfig(key, DEFAULT_TIMEOUT); } @Override - public void addConfigListener(String key) { - LOGGER.info("ZK添加监听器, key:{}", key); - if (StringUtil.isBlank(key)) { - return; + public boolean removeConfig(String key, long timeoutMills) { + PROPERTIES.remove(key); + try { + zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigByte(PROPERTIES)); + } catch (Exception e) { + return false; } - ZkListener zkListener = new ZkListener("/"+key); - CONFIG_LISTENERS_MAP.put(key, zkListener); - zkListener.onProcessEvent(new ConfigChangeEvent()); + return true; } + + @Override - public void removeConfigListener(String key) { - if (StringUtil.isBlank(key)) { - return; + public void addConfigListener(String node) { + if(!DEFAULT_CONFIG_PATH.equals(node)){ + throw new RuntimeException("no support"); } - LOGGER.info("ZK删除监听器, key:{}", key); - ConfigChangeListener configChangeListeners = getConfigListeners(key); - CONFIG_LISTENERS_MAP.remove(key); - ZkListener zkListener = (ZkListener) configChangeListeners; - zkListener.onShutDown(); + LOGGER.info("ZK添加监听器, node:{}", node); + ZKLISTENER = new ZkListener(node); + ZKLISTENER.onProcessEvent(new ConfigChangeEvent()); } @Override - public ConfigChangeListener getConfigListeners(String key) { - return CONFIG_LISTENERS_MAP.get(key); + public void removeConfigListener(String node) { + if(!DEFAULT_CONFIG_PATH.equals(node)){ + throw new RuntimeException("no support"); + } + LOGGER.info("ZK删除监听器, node:{}", node); + ZKLISTENER.onShutDown(); + ZKLISTENER = null; } @Override - public Map getConfigByPrefix(String prefix) { + public List getConfigByPrefix(String prefix) { + try { + String val = getConfig(prefix); + return FastJsonUtils.toList(val,String.class); + } catch (Exception e) { + e.printStackTrace(); + } return null; } + @Override public String getType() { return "zk"; @@ -164,29 +175,14 @@ public class ZkConfigurator implements Configurator { - static void loadZkData() throws Exception { byte[] bt = zkClient.getData().forPath(DEFAULT_CONFIG_PATH); if (bt != null && bt.length > 0){ ByteArrayInputStream bArray = new ByteArrayInputStream(bt); - pros.load(bArray); + PROPERTIES.load(bArray); } + LOGGER.info("# loadZkData # PROPERTIES:{}",JSON.toJSONString(PROPERTIES)); } - - private static byte[] formatConfigStr() { - StringBuilder sb = new StringBuilder(); - - Enumeration enumeration = pros.propertyNames(); - while (enumeration.hasMoreElements()) { - String key = (String) enumeration.nextElement(); - String property = pros.getProperty(key); - sb.append(key).append("=").append(property).append("\n"); - } - LOGGER.info("ZK更新配置文件:{}", sb.toString()); - return sb.toString().getBytes(); - } - - } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java index d0d2ba7..c4ac939 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java @@ -10,7 +10,6 @@ package com.jd.platform.jlog.zk; public class ZkConstant { static final String NAMESPACE = "jLog"; - static final String ROOT = "/jLog"; - static final String SERVER_ADDR_KEY = "serverAddr"; static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; + static final String DEFAULT_WORKER_PATH = "/workers"; } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java index 29cc595..388b666 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -1,6 +1,7 @@ package com.jd.platform.jlog.zk; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.ConfigChangeType; @@ -12,6 +13,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; import java.util.Properties; +import java.util.Set; import static com.jd.platform.jlog.zk.ZkConfigurator.*; @@ -25,7 +27,10 @@ public class ZkListener implements ConfigChangeListener { private NodeCache cache; + private String path; + public ZkListener(String path) { + this.path = path; cache = new NodeCache(zkClient, path); LOGGER.info("构造ZkListener:{}",path); try { @@ -57,34 +62,21 @@ public class ZkListener implements ConfigChangeListener { @Override public void onChangeEvent(ConfigChangeEvent event) { - LOGGER.info("ZK数据变更,通用事件触发onChangeEvent",event.toString()); - changeEvent(); - } - - - private void changeEvent(){ - ChildData data = cache.getCurrentData(); - LOGGER.info("变更-当前数据cache:{}", cache); - Properties propsTmp = pros; + LOGGER.info("ZK数据变更-当前监听器关注的path:{}", path); + Properties propsTmp = new Properties(PROPERTIES); try { LOGGER.info("ZK数据变更,旧Properties:{}", JSON.toJSONString(propsTmp)); loadZkData(); - LOGGER.info("ZK数据变更,新Properties:{}", JSON.toJSONString(pros)); + LOGGER.info("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES)); } catch (Exception e) { e.printStackTrace(); } - LOGGER.info("ZK数据变更,CONFIG_LISTENERS_MAP:{}", JSON.toJSONString(CONFIG_LISTENERS_MAP)); - for (Map.Entry entry : CONFIG_LISTENERS_MAP.entrySet()) { - String listenedKey = entry.getKey(); - String propertyOld = pros.getProperty(listenedKey); - String propertyNew = propsTmp.getProperty(listenedKey); - if (!propertyOld.equals(propertyNew)) { - ConfigChangeEvent event = new ConfigChangeEvent() - .setKey(listenedKey) - .setNewValue(propertyNew) - .setChangeType(ConfigChangeType.MODIFY); - entry.getValue().onProcessEvent(event); + Set diffKeys = CollectionUtil.diffKeys(propsTmp, PROPERTIES); + if(!diffKeys.isEmpty()){ + for (String diffKey : diffKeys) { + LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey, new ConfigChangeEvent(diffKey, propsTmp.get(diffKey).toString(), PROPERTIES.get(diffKey).toString())); } } } + } -- Gitee From a3dfef58aa9ada905d1bd46aec732d1c547bb7fc Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Wed, 2 Mar 2022 16:37:24 +0800 Subject: [PATCH 14/22] =?UTF-8?q?todo=20=20=E5=8C=85=E8=A3=85=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/apollo/ApolloConfigurator.java | 7 ------ .../platform/jlog/etcd/EtcdConfigurator.java | 23 +++++++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index 6c5a889..803bd32 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -1,7 +1,6 @@ package com.jd.platform.jlog.config.apollo; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.concurrent.*; @@ -129,12 +128,6 @@ public class ApolloConfigurator implements Configurator { } - @Override - public ConfigChangeListener getConfigListeners(String dataId) { - return CONFIG_LISTENER_MAP.get(dataId); - } - - @Override public String getType() { return "apollo"; diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index bf55fcb..cc2a5c1 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -16,6 +16,9 @@ import com.jd.platform.jlog.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte; +import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; +import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; /** @@ -35,9 +38,8 @@ public class EtcdConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; - private static final String ROOT = "/jLog"; - private static final String PROPERTIES_PATH = "/properties"; + private static final String PROPERTIES_PATH = "/jLog/properties"; private static Properties PROPERTIES = new Properties(); @@ -47,7 +49,7 @@ public class EtcdConfigurator implements Configurator { private EtcdConfigurator() { LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); client = EtcdClient.forEndpoints(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,2000L)).withPlainText().build(); - RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(ROOT + PROPERTIES_PATH)).sync(); + RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync(); List keyValues = rangeResponse.getKvsList(); if (CollectionUtil.isEmpty(keyValues)) { return; @@ -87,26 +89,29 @@ public class EtcdConfigurator implements Configurator { @Override public boolean putConfig(String key, String content) { - client.getKvClient().put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(content)).sync(); - return true; + return putConfig(key, content, DEFAULT_TIMEOUT); } @Override public boolean putConfig(String key, String content, long timeoutMills) { - client.getKvClient().put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(content)).timeout(timeoutMills).sync(); + if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){ + return false; + } + PROPERTIES.setProperty(key, content); + client.getKvClient().put(ByteString.copyFromUtf8(PROPERTIES_PATH), ByteString.copyFromUtf8(formatConfigStr(PROPERTIES))).sync(); return true; } @Override public boolean removeConfig(String key) { - client.getKvClient().delete(ByteString.copyFromUtf8(key)).sync(); - return true; + return removeConfig(key, DEFAULT_TIMEOUT); } @Override public boolean removeConfig(String key, long timeoutMills) { - client.getKvClient().delete(ByteString.copyFromUtf8(key)).timeout(timeoutMills).sync(); + PROPERTIES.remove(key); + client.getKvClient().put(ByteString.copyFromUtf8(PROPERTIES_PATH), ByteString.copyFromUtf8(formatConfigStr(PROPERTIES))).sync(); return true; } -- Gitee From 92c1b13ce1a700fd52bcf2c0cf19ab6709934f06 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Wed, 9 Mar 2022 23:40:20 +0800 Subject: [PATCH 15/22] =?UTF-8?q?properties=E5=8F=8D=E5=B0=84=E8=AF=BB?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 14 +- .../jlog/client/TracerClientStarter.java | 37 ++--- clientdemo/src/main/resources/jLog.properties | 1 - .../jlog/common/constant/Constant.java | 5 + .../platform/jlog/common/tag/CollectMode.java | 55 +++++-- .../platform/jlog/common/tag/TagConfig.java | 47 +++--- .../platform/jlog/common/tag/TagHandler.java | 35 +++-- .../jlog/common/utils/CollectionUtil.java | 86 ++++++++++- .../jlog/common/utils/ConfigUtil.java | 74 ++++++++- .../config/apollo/ApolloConfigurator.java | 20 +++ .../jd/platform/jlog/core/Configurator.java | 26 +++- .../jlog/core/ConfiguratorFactory.java | 19 ++- .../com/jd/platform/jlog/core/Constant.java | 4 +- .../platform/jlog/core/FileConfigurator.java | 71 +++++---- .../com/jd/platform/jlog/core/FileNode.java | 30 ++++ .../jd/platform/jlog/core/JcProperties.java | 139 +++++++++++++++++ .../com/jd/platform/jlog/core/NodeBak.java | 142 ++++++++++++++++++ .../com/jd/platform/jlog/core/ObjObj.java | 45 ++++++ .../com/jd/platform/jlog/core/Server.java | 32 ++++ .../platform/jlog/core/TagHandlerBuilder.java | 39 +++++ .../platform/jlog/etcd/EtcdConfigurator.java | 30 +++- .../jd/platform/jlog/etcd/EtcdListener.java | 24 +-- .../jlog/nacos/NacosConfigurator.java | 35 ++++- .../jd/platform/jlog/nacos/NacosConstant.java | 1 - .../com/jd/platform/jlog/nacos/NacosTest.java | 4 +- .../jd/platform/jlog/zk/ZkConfigurator.java | 42 +++--- .../com/jd/platform/jlog/zk/ZkListener.java | 17 ++- .../java/com/jd/platform/jlog/zk/ZkTest.java | 3 - {clientdemo => example}/pom.xml | 6 +- .../jlog/clientdemo/ExampleApplication.java | 4 +- .../appender/TracerLogbackAppender.java | 6 +- .../jlog/clientdemo/config/DemoConfig.java | 0 .../jlog/clientdemo/web/TestController.java | 31 +++- .../platform/jlog/clientdemo/web/TestReq.java | 0 .../src/main/resources/application.properties | 16 ++ .../src/main/resources/application.ymlbak | 0 .../src/main/resources/bakapplication.yml | 7 +- example/src/main/resources/bakjLog.properties | 6 + .../src/main/resources/logback-spring.xml | 0 .../com/jd/platform/jlog/test/Common.java | 4 +- .../jlog/test/EtcdConfiguratorTest.java | 91 +++++++++++ .../jlog/test/FileConfiguratorTest.java | 32 ++-- .../jlog/test/NacosConfiguratorTest.java | 87 +++++++++++ .../platform/jlog/test/TracerPacketTest.java | 4 +- .../jlog/test/ZKConfiguratorTest.java | 37 +++-- pom.xml | 2 +- 46 files changed, 1170 insertions(+), 240 deletions(-) delete mode 100644 clientdemo/src/main/resources/jLog.properties create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java rename {clientdemo => example}/pom.xml (96%) rename clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java => example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java (79%) rename {clientdemo => example}/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java (98%) rename {clientdemo => example}/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java (100%) rename {clientdemo => example}/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java (56%) rename {clientdemo => example}/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java (100%) create mode 100644 example/src/main/resources/application.properties rename {clientdemo => example}/src/main/resources/application.ymlbak (100%) rename clientdemo/src/main/resources/application.yml => example/src/main/resources/bakapplication.yml (73%) create mode 100644 example/src/main/resources/bakjLog.properties rename {clientdemo => example}/src/main/resources/logback-spring.xml (100%) rename {clientdemo => example}/src/test/java/com/jd/platform/jlog/test/Common.java (93%) create mode 100644 example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java rename {clientdemo => example}/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java (63%) create mode 100644 example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java rename {clientdemo => example}/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java (95%) rename {clientdemo => example}/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java (69%) diff --git a/client/pom.xml b/client/pom.xml index 26c0923..6de7c96 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,19 +18,17 @@ 1.4-SNAPSHOT - + + - - - + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index d252b50..1f72fb1 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -1,15 +1,13 @@ package com.jd.platform.jlog.client; +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.task.Monitor; -import com.jd.platform.jlog.client.udp.HttpSender; -import com.jd.platform.jlog.client.udp.UdpClient; -import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.tag.TagConfig; -import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; +import com.jd.platform.jlog.core.TagHandlerBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,45 +80,34 @@ public class TracerClientStarter { * 启动监听 */ public void startPipeline() { - // 校验和设置 - checkAndSetTagConfig(); - - // TagHandler.build(tagConfig); + // 初始化配置 + initJLogConfig(); Context.MDC = mdc; Monitor starter = new Monitor(); starter.start(); - UdpClient udpClient = new UdpClient(); + /* UdpClient udpClient = new UdpClient(); udpClient.start(); //开启发送 UdpSender.uploadToWorker(); //开启大对象http发送 - HttpSender.uploadToWorker(); + HttpSender.uploadToWorker();*/ } /** * 如果未赋值，从配置器获取，底层是Properties */ - private void checkAndSetTagConfig(){ + private void initJLogConfig(){ + LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig)); Configurator configurator = ConfiguratorFactory.getInstance(); - if(tagConfig != null){ - LOGGER.info("从主配置获取的tagConfig:{}", tagConfig.toString()); - configurator.addConfigListener("/application.yml"); - return; - } - String reqTag = configurator.getConfig("reqTags"); - String logTag = configurator.getConfig("logTags"); - String regex = configurator.getConfig("regex"); - String delimiter = configurator.getConfig("delimiter"); - String join = configurator.getConfig("join"); - tagConfig = TagConfig.Builder.aTagConfig().reqTags(FastJsonUtils.toList(reqTag, String.class)) - .logTags(FastJsonUtils.toList(logTag, String.class)) - .regex(regex).delimiter(delimiter).join(join).build(); - LOGGER.info("从配置器获取的tagConfig:{}", tagConfig.toString()); + TagHandlerBuilder.buildTagHandler(tagConfig, configurator); + configurator.addConfigListener("/application.properties"); + // configurator.addConfigListener("/application.yml"); + } } diff --git a/clientdemo/src/main/resources/jLog.properties b/clientdemo/src/main/resources/jLog.properties deleted file mode 100644 index 3dd4dd0..0000000 --- a/clientdemo/src/main/resources/jLog.properties +++ /dev/null @@ -1 +0,0 @@ -reqTags=["AAA","CCC"] \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index 8673867..1c069f4 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -34,6 +34,11 @@ public class Constant { */ public static int TAG_NORMAL_KEY_MAX_LEN = 20; + /** + * 可供提取的日志最小长度 + */ + public static int EXTRACT_MIN_LEN = 5; + /** * 符合正则但不构成kv结构的普通日志，如「xx查询为空」 */ diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java index 5e1646c..191ece5 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java @@ -10,6 +10,11 @@ package com.jd.platform.jlog.common.tag; public class CollectMode { + /** + * 挂起 不提取不压缩 + */ + public static final long SUSPEND = 0L; + /** * 提取req */ @@ -31,12 +36,12 @@ public class CollectMode { public static final long C_LOG = 1L << 4; /** - * 提取log + * 提取resp */ public static final long E_RESP = 1L << 5; /** - * 压缩log + * 压缩resp */ public static final long C_RESP = 1L << 6; @@ -44,30 +49,56 @@ public class CollectMode { /** ======================================= 下面是组合 ======================================= */ + + + /** + * 提取req+log + */ + public static final long EXTRACT_REQ_LOG = E_REQ | E_LOG; + /** - * 提取req，压缩req + * 提取req+resp */ - public static final long E_REQ_C_REQ = E_REQ | C_REQ; + public static final long EXTRACT_REQ_REQP = E_REQ | E_RESP; /** - * 提取log，压缩log + * 提取req+resp */ - public static final long E_LOG_C_LOG = E_LOG | C_LOG; + public static final long EXTRACT_REQ_RESP = E_REQ | E_RESP; + + /** + * 提取req+log+resp + */ + public static final long EXTRACT_ALL = E_REQ | E_LOG | E_RESP; + /** - * 提取req+log，压缩req + * 压缩req+log */ - public static final long E_REQ_E_LOG_C_REQ = E_REQ | E_LOG | C_REQ; + public static final long COMPRESS_REQ_LOG = C_REQ | C_LOG; /** - * 提取req+log，压缩log + * 压缩req+resp */ - public static final long E_REQ_E_LOG_C_LOG = E_REQ | E_LOG | C_LOG; + public static final long COMPRESS_REQ_REQP = C_REQ | C_RESP; /** - * 提取req+log，压缩req+log + * 压缩req+resp */ - public static final long E_REQ_E_LOG_C_REQ_E_LOG = E_REQ | E_LOG | C_REQ | E_LOG; + public static final long COMPRESS_REQ_C_RESP = C_REQ | C_RESP; + + /** + * 压缩req+log+resp + */ + public static final long COMPRESS_ALL = C_REQ | C_LOG | C_RESP; + + public static void main(String[] args) { + System.out.println(isMatched(EXTRACT_REQ_LOG, EXTRACT_REQ_RESP)); + } + public static boolean isMatched(long indicator, long position) { + return (indicator & position) == position; + } } + // curl --location --request POST 'http://10.96.98.110:8058/app/flow/getappraiselist' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'token=DF9YcBVAaMiLCZtZEZCf6MunJX8I3jWGmQXwRzb-oQIszD1uAjEQgNG7fG1Gq5nx2I6nzBFygwSWn8ZIIKrV3h0hUb3ubUwlKYsuijCNNGE6WVVLCLOQ1uvoLTza6KrCDPJNJfn5RfgjQfgnw8vo7j6KqX-HC0dyCCu58bg974f1U-_CibQWrXlXC-FM8lWrqhZrvVYvCBcSQ7iSur8CAAD__w==' --data-urlencode 'cityID=55000116' --data-urlencode 'appVersion=2.0.28' --data-urlencode 'country=BR' \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java index c71d20a..c6e53a7 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java @@ -3,6 +3,9 @@ package com.jd.platform.jlog.common.tag; import java.io.Serializable; import java.util.List; +import static com.jd.platform.jlog.common.tag.CollectMode.COMPRESS_ALL; +import static com.jd.platform.jlog.common.tag.CollectMode.EXTRACT_ALL; + /** * @author tangbohu * @version 1.0.0 @@ -37,14 +40,14 @@ public class TagConfig implements Serializable { private String join = "="; /** - * 提取入参开关 + * 提取策略 */ - private Boolean extractReq = true; + private long extract = EXTRACT_ALL; /** - * 提取普通log开关 + * 压缩策略 */ - private Boolean extractLog = true; + private long compress = COMPRESS_ALL; public List getReqTags() { @@ -87,20 +90,20 @@ public class TagConfig implements Serializable { this.join = join; } - public Boolean getExtractReq() { - return extractReq; + public long getExtract() { + return extract; } - public void setExtractReq(Boolean extractReq) { - this.extractReq = extractReq; + public void setExtract(long extract) { + this.extract = extract; } - public Boolean getExtractLog() { - return extractLog; + public long getCompress() { + return compress; } - public void setExtractLog(Boolean extractLog) { - this.extractLog = extractLog; + public void setCompress(long compress) { + this.compress = compress; } public static final class Builder { @@ -109,8 +112,8 @@ public class TagConfig implements Serializable { private String regex; private String delimiter; private String join; - private Boolean extractReq; - private Boolean extractLog; + private long extract; + private long compress; public Builder() { } @@ -144,13 +147,13 @@ public class TagConfig implements Serializable { return this; } - public Builder extractReq(Boolean extractReq) { - this.extractReq = extractReq; + public Builder extract(long extract) { + this.extract = extract; return this; } - public Builder extractLog(Boolean extractLog) { - this.extractLog = extractLog; + public Builder compress(long compress) { + this.compress = compress; return this; } @@ -161,8 +164,8 @@ public class TagConfig implements Serializable { tagConfig.setRegex(regex); tagConfig.setDelimiter(delimiter); tagConfig.setJoin(join); - tagConfig.setExtractReq(extractReq); - tagConfig.setExtractLog(extractLog); + tagConfig.setExtract(extract); + tagConfig.setCompress(compress); return tagConfig; } } @@ -175,8 +178,8 @@ public class TagConfig implements Serializable { ", regex='" + regex + '\'' + ", delimiter='" + delimiter + '\'' + ", join='" + join + '\'' + - ", extractReq='" + extractReq + '\'' + - ", extractLog='" + extractLog + '\'' + + ", extract='" + extract + '\'' + + ", compress='" + compress + '\'' + '}'; } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java index f037e63..831a7bf 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java @@ -12,8 +12,12 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.jd.platform.jlog.common.constant.Constant.EXTRACT_MIN_LEN; import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY; import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY_MAX_LEN; +import static com.jd.platform.jlog.common.tag.CollectMode.*; +import static com.jd.platform.jlog.common.tag.CollectMode.E_LOG; +import static com.jd.platform.jlog.common.tag.CollectMode.E_REQ; import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; /** @@ -28,12 +32,8 @@ public class TagHandler { private Set reqTags; - private boolean extractReq; - private Set logTags; - private boolean extractLog; - private String delimiter = "|"; private int delimiterLen = delimiter.length(); @@ -42,22 +42,25 @@ public class TagHandler { private Pattern pattern; + private long extract; + + private long compress; + private static volatile TagHandler INSTANCE = null; /** * 构建标签处理器 * @param tagConfig 配置类 */ - public static void build(TagConfig tagConfig) { + public static void buildHandler(TagConfig tagConfig) { - if(!tagConfig.getExtractReq() && !tagConfig.getExtractLog()){ + if(tagConfig.getExtract() == SUSPEND && tagConfig.getCompress() == SUSPEND ){ return; } TagHandler handler = new TagHandler(); - - handler.extractReq = tagConfig.getExtractReq(); - handler.extractLog = tagConfig.getExtractLog(); + handler.extract = tagConfig.getExtract(); + handler.compress = tagConfig.getCompress(); handler.reqTags = new HashSet<>(tagConfig.getReqTags()); handler.logTags = new HashSet<>(tagConfig.getLogTags()); @@ -85,7 +88,9 @@ public class TagHandler { */ public static Map extractReqTag(Map params, @NotNull Map ext) { - if(INSTANCE == null || !INSTANCE.extractReq){ return null; } + if(INSTANCE == null || !isMatched(INSTANCE.extract, E_REQ)){ return null; } + + System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(INSTANCE.reqTags)); Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); for (String tag : INSTANCE.reqTags) { @@ -99,7 +104,7 @@ public class TagHandler { requestMap.put(tag, params.get(tag)[0]); } } - + System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(requestMap)); return requestMap; } @@ -110,7 +115,7 @@ public class TagHandler { * @return tags */ public static Map extractLogTag(String content) { - if(INSTANCE == null || !INSTANCE.extractLog || content.length() < 1){ + if(INSTANCE == null || !isMatched(INSTANCE.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ return null; } @@ -131,6 +136,7 @@ public class TagHandler { } } } + System.out.println("提取到了请求log日志标签："+JSON.toJSONString(tagMap)); return tagMap; } @@ -142,7 +148,7 @@ public class TagHandler { */ public synchronized static void refresh(TagConfig tagConfig) { INSTANCE = null; - build(tagConfig); + buildHandler(tagConfig); } @@ -154,6 +160,9 @@ public class TagHandler { ", delimiter='" + delimiter + '\'' + ", delimiterLen=" + delimiterLen + ", join='" + join + '\'' + + ", pattern=" + pattern + + ", extract=" + extract + + ", compress=" + compress + '}'; } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java index 5890083..87af2f6 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java @@ -133,13 +133,87 @@ public class CollectionUtil { public static void main(String[] args) { - HashMap m1 = new HashMap<>(); - m1.put("t1",1); - m1.put("t2",2); + int fail = 0; + int count = 10000; + Set set = new HashSet<>(); + for (int i = 0; i < count; i++) { + int h = randomMac4Qemu().hashCode(); + int val = indexFor(h); + boolean r = set.add(val); + // System.out.println(val); + if(!r){ + fail++; + } + } + + int fail2 = 0; + Set set2 = new HashSet<>(); + for (int i = 0; i < count; i++) { + int h = randomMac4Qemu().hashCode(); + Random rd = new Random(h); + + int val = rd.nextInt(16385); + boolean r = set.add(val); + System.out.println(val); + if(!r){ + fail2++; + } + } + + System.out.println("11--》 "+fail); + System.out.println("22-》 "+ fail2); + } + static int indexFor(int h) { + return h & (16384-1); + } + public static String randomMac4Qemu() { + /* if(1==1){ + return getRandomIp(); + }*/ + Random random = new Random(); + String[] mac = { + /* String.format("%02x", 0x52), + String.format("%02x", 0x54), + String.format("%02x", 0x00),*/ + String.format("%02x", random.nextInt(0xff)), + String.format("%02x", random.nextInt(0xff)), + String.format("%02x", random.nextInt(0xff)) + }; + return String.join(":", mac); + } + + + public static String getRandomIp(){ + + //ip范围 + int[][] range = {{607649792,608174079},//36.56.0.0-36.63.255.255 + {1038614528,1039007743},//61.232.0.0-61.237.255.255 + {1783627776,1784676351},//106.80.0.0-106.95.255.255 + {2035023872,2035154943},//121.76.0.0-121.77.255.255 + {2078801920,2079064063},//123.232.0.0-123.235.255.255 + {-1950089216,-1948778497},//139.196.0.0-139.215.255.255 + {-1425539072,-1425014785},//171.8.0.0-171.15.255.255 + {-1236271104,-1235419137},//182.80.0.0-182.92.255.255 + {-770113536,-768606209},//210.25.0.0-210.47.255.255 + {-569376768,-564133889}, //222.16.0.0-222.95.255.255 + }; + + Random rdint = new Random(); + int index = rdint.nextInt(10); + return num2ip(range[index][0]+new Random().nextInt(range[index][1]-range[index][0])); + } + + + public static String num2ip(int ip) { + int [] b=new int[4] ; + String x = ""; - HashMap m2 = new HashMap<>(); - m2.put("t2",2); - m2.put("t3",3); + b[0] = (int)((ip >> 24) & 0xff); + b[1] = (int)((ip >> 16) & 0xff); + b[2] = (int)((ip >> 8) & 0xff); + b[3] = (int)(ip & 0xff); + x=Integer.toString(b[0])+"."+Integer.toString(b[1])+"."+Integer.toString(b[2])+"."+Integer.toString(b[3]); + return x; } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java index e74de5e..4e27fb8 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -2,8 +2,12 @@ package com.jd.platform.jlog.common.utils; import com.jd.platform.jlog.common.constant.Constant; -import java.util.Enumeration; -import java.util.Properties; +import java.beans.IntrospectionException; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; import java.util.concurrent.ThreadLocalRandom; /** @@ -18,13 +22,14 @@ public class ConfigUtil { private static final String SERVER_SUFFIX = "Server"; - public static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); + public static final char MID_LINE = '-'; + public static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); public static String escapeExprSpecialWord(String str) { - if(str != null && str.length() > 0){ + if (str != null && str.length() > 0) { for (String s : Constant.SPECIAL_CHAR) { if (str.contains(s)) { str = str.replace(s, "\\" + s); @@ -35,7 +40,6 @@ public class ConfigUtil { } - public static String formatConfigStr(Properties properties) { StringBuilder sb = new StringBuilder(); @@ -43,7 +47,7 @@ public class ConfigUtil { while (enumeration.hasMoreElements()) { String key = (String) enumeration.nextElement(); Object property = properties.get(key); - if(property != null){ + if (property != null) { property = String.valueOf(property); } sb.append(key).append("=").append(property).append("\n"); @@ -52,12 +56,41 @@ public class ConfigUtil { } - public static byte[] formatConfigByte(Properties properties) { return formatConfigStr(properties).getBytes(); } + //递归解析map对象 + public static void loadRecursion(Map map, String key, Properties props) { + map.forEach((k, v) -> { + System.out.println("k => " + k + " v.class => " + v.getClass() + " val => " + v.toString() + " 是不是爸爸 => " + isParent(v)); + if (isParent(v)) { + if (v instanceof Boolean || v instanceof List) { + props.put(key + "." + k, v); + } else { + Map nextValue = (Map) v; + loadRecursion(nextValue, (("".equals(key) ? "" : key + ".") + k), props); + } + } else { + props.put(key + "." + k, v); + } + }); + } + + //判断是否还有子节点 + public static boolean isParent(Object o) { + if (!(o instanceof String || o instanceof Character || o instanceof Byte)) { + try { + Number n = (Number) o; + // System.out.println("isParent ==> "+n); + } catch (Exception e) { + return true; + } + } + return false; + } + /* public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception { @@ -86,4 +119,31 @@ public class ConfigUtil { items[0] = (byte) ((char) items[0] - 'a' + 'A'); return new String(items); } + + + + + public static String camelToMidline(String param) { + if (param == null || "".equals(param.trim())) { + return ""; + } + int len = param.length(); + StringBuilder sb = new StringBuilder(len); + for (int i = 0; i < len; i++) { + char c = param.charAt(i); + if (Character.isUpperCase(c)) { + sb.append("-"); + sb.append(Character.toLowerCase(c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } + + public static String lowerFirst(String fromStr){ + char[] chars = fromStr.toCharArray(); + chars[0] += 32; + return String.valueOf(chars); + } } diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index 803bd32..f0e6432 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -77,6 +77,26 @@ public class ApolloConfigurator implements Configurator { return instance; } + @Override + public String getString(String key) { + return null; + } + + @Override + public Long getLong(String key) { + return null; + } + + @Override + public List getList(String key) { + return null; + } + + @Override + public T getObject(String key, Class clz) { + return null; + } + @Override public String getConfig(String key) { return config.getProperty(key,""); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index 36df609..4ae742d 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -1,7 +1,6 @@ package com.jd.platform.jlog.core; import java.util.List; -import java.util.Map; /** * @author tangbohu @@ -13,19 +12,34 @@ import java.util.Map; public interface Configurator { /** - * 获取配置 + * 获取string配置 * @param key key * @return val */ - String getConfig(String key); + String getString(String key); + /** - * 获取配置 + * 获取LONG配置 + * @param key key + * @return val + */ + Long getLong(String key); + + /** + * 获取LIST类型配置 * @param key key - * @param timeoutMills timeoutMills * @return val */ - String getConfig(String key, long timeoutMills); + List getList(String key); + + /** + * 获取实体类型配置 + * @param key key + * @return val + */ + T getObject(String key, Class clz); + /** * 设置配置 diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java index 92e7795..c87087f 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -32,8 +32,8 @@ public class ConfiguratorFactory { synchronized (Configurator.class) { if (instance == null) { instance = buildConfiguration(); - LOGGER.info("构建总配置器单例完成 instance 获取reqTags结果:{}", instance.getConfig("reqTags")); - LOGGER.info("构建总配置器单例完成 Base 获取serverAddr结果:{}", base.getConfig("serverAddr")); + LOGGER.info("构建总配置器单例完成 instance 获取类型结果:{}", instance.getType()); + LOGGER.info("构建总配置器单例完成 Base 获取serverAddr结果:{}", base.getString("serverAddr")); } } } @@ -69,4 +69,19 @@ public class ConfiguratorFactory { return base; } + + + private static Configurator refreshBaseConfiguration() { + + synchronized (Configurator.class){ + try { + base = new FileConfigurator(); + } catch (IOException e) { + LOGGER.info("文件配置器构建失败", e); + throw new RuntimeException("build file buildConfiguration fail", e); + } + } + return base; + } + } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java index f40ea23..fb05839 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java @@ -15,7 +15,7 @@ public class Constant { /** * 监听重读配置文件间隔 单位ms */ - static final long LISTENER_CONFIG_INTERVAL = 10000; + static final long LISTENER_CONFIG_INTERVAL = 3000; static final String CONFIG_FILE_PROPERTIES = "/application.properties"; @@ -34,7 +34,7 @@ public class Constant { public static final String SERVER_ADDR_KEY = "serverAddr"; - + public static final String NAMESPACE_KEY = "namespace"; /** * 配置文件集合 diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index 20730b1..26450e5 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -28,7 +28,7 @@ public class FileConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class); - private static Properties PROPERTIES = new Properties(); + private static JcProperties PROPERTIES = new JcProperties(); private static volatile FileListener FILELISTENER = null; @@ -44,18 +44,19 @@ public class FileConfigurator implements Configurator { for (String file : CONFIG_FILES) { String fileName = StringUtil.isEmpty(env) ? file : file + "_" + env; URL url = this.getClass().getResource(file); - if(url != null){ - try (InputStream is = url.openStream()) { - Properties props = new Properties(); - if (fileName.contains(YML)) { - props.putAll(new Yaml().loadAs(is, Map.class)); - } else { - props.load(is); - } - FILE_MODIFY_MAP.put(fileName, new FileWrapper(new File(url.getFile()).lastModified(), props)); - PROPERTIES.putAll(props); - LOGGER.info("{}配置文件配置:{}", file, props.toString()); + if(url == null){ + continue; + } + try (InputStream is = url.openStream()) { + JcProperties props = new JcProperties(); + if (fileName.contains(YML)) { + props.putAll(new Yaml().loadAs(is, Map.class)); + } else { + props.load(is); } + FILE_MODIFY_MAP.put(fileName, new FileWrapper(new File(url.getFile()).lastModified(), props)); + PROPERTIES.putAll(props); + LOGGER.info("{}配置文件配置:{}", file, props.toString()); } } LOGGER.info("合并后的配置:{}",PROPERTIES.toString()); @@ -63,20 +64,24 @@ public class FileConfigurator implements Configurator { @Override - public String getConfig(String key) { - Object val = PROPERTIES.get(key); - if(val != null){ - return String.valueOf(val); - } - return null; + public String getString(String key) { + return PROPERTIES.getString(key); } + @Override + public Long getLong(String key) { + return PROPERTIES.getLong(key); + } @Override - public String getConfig(String key, long timeoutMills) { - return PROPERTIES.getProperty(key); + public List getList(String key) { + return PROPERTIES.getStrList(key); } + @Override + public T getObject(String key, Class clz) { + return PROPERTIES.getBean(key, clz); + } @Override public boolean putConfig(String key, String content) { return false; } @@ -106,7 +111,6 @@ public class FileConfigurator implements Configurator { return; } String env = System.getenv(ENV); - for (String file : CONFIG_FILES) { file = StringUtil.isEmpty(env) ? file : file + "_" + env; if(node.equals(file)){ @@ -171,7 +175,8 @@ public class FileConfigurator implements Configurator { TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("fileListener", 1)); - FileListener() { } + FileListener() { + } synchronized void addListener() { FILELISTENER.onProcessEvent(new ConfigChangeEvent()); @@ -202,28 +207,30 @@ public class FileConfigurator implements Configurator { * 检测文件最后修改时间 和重载文件 */ private void checkAndConfigure(){ + AtomicBoolean change = new AtomicBoolean(false); Map newModifyMap = checkAndReload(); + FILE_MODIFY_MAP.forEach((k, v)->{ FileWrapper newFile = newModifyMap.get(k); if(newFile != null && newFile.change){ - Set diffKeys = CollectionUtil.diffKeys(newFile.props, v.props); if(!diffKeys.isEmpty()){ change.set(true); - v.props = newFile.props; - for (String diffKey : diffKeys) { - LOGGER.warn("文件 {} 配置变更 key={}变更事件:{}", k, diffKey, new ConfigChangeEvent(diffKey, v.props.get(diffKey).toString(), newFile.props.get(diffKey).toString())); + LOGGER.warn("文件 {} 配置变更 key={}变更事件:{}", k, diffKey, new ConfigChangeEvent(diffKey, String.valueOf(v.props.get(diffKey)), String.valueOf(newFile.props.get(diffKey)))); } + v.props = newFile.props; + v.lastModify= newFile.lastModify; } } }); if(change.get()){ - LOGGER.info("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); + // LOGGER.info("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); PROPERTIES.clear(); FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props)); LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); + TagHandlerBuilder.refresh(); } } @@ -234,8 +241,10 @@ public class FileConfigurator implements Configurator { private Map checkAndReload() { Map fileWrapperMap = new ConcurrentHashMap<>(3); + for (String fileName : LISTENED_FILES) { URL url = this.getClass().getResource(fileName); + if(url == null){ continue; } @@ -246,7 +255,7 @@ public class FileConfigurator implements Configurator { if(curLastMod <= cacheLastMod){ continue; } - Properties props = new Properties(); + JcProperties props = new JcProperties(); try (InputStream is = url.openStream()) { if (fileName.contains(YML)) { props.putAll(new Yaml().loadAs(is, Map.class)); @@ -269,14 +278,14 @@ public class FileConfigurator implements Configurator { private boolean change; - private Properties props; + private JcProperties props; - FileWrapper(long lastModify, Properties props) { + FileWrapper(long lastModify, JcProperties props) { this.lastModify = lastModify; this.change = false; this.props = props; } - FileWrapper(long lastModify, boolean change, Properties props) { + FileWrapper(long lastModify, boolean change, JcProperties props) { this.lastModify = lastModify; this.change = change; this.props = props; diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java new file mode 100644 index 0000000..49634af --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -0,0 +1,30 @@ +package com.jd.platform.jlog.core; + +import com.alibaba.fastjson.JSON; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName FileNode.java + * @Description TODO + * @createTime 2022年03月08日 16:32:00 + */ +public final class FileNode { + + private String fillPath; + + private long lastModity; + + public static void main(String[] args) throws IOException { + JcProperties properties = new JcProperties(); + String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; + properties.load(new FileInputStream(path)); + System.out.println(JSON.toJSONString(properties)); + Server bean = properties.getBean("server", Server.class); + System.out.println(JSON.toJSONString(bean)); + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java new file mode 100644 index 0000000..8bd02bd --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java @@ -0,0 +1,139 @@ +package com.jd.platform.jlog.core; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.ConfigUtil; +import com.jd.platform.jlog.common.utils.FastJsonUtils; +import com.jd.platform.jlog.common.utils.StringUtil; + +import java.beans.IntrospectionException; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; + +import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName JcProperties.java + * @Description TODO + * @createTime 2022年03月07日 12:43:00 + */ +public class JcProperties extends Properties { + + public JcProperties() { + } + + public String getString(String key) { + Object val = super.get(key); + if(val == null){ + return null; + } + return String.valueOf(super.get(key)); + } + + public Long getLong(String key) { + String val = getString(key); + if(StringUtil.isEmpty(val)){ + return null; + } + return Long.valueOf(val); + } + + public List getStrList(String key) { + String val = getString(key); + if(StringUtil.isEmpty(val)){ + return null; + } + return FastJsonUtils.toList(val, String.class); + } + + public T getBean(String key, Class clz) { + T bean = FastJsonUtils.toBean(JSON.toJSONString(get(key)), clz); + if(bean != null){ + return bean; + } + try { + T instance = clz.newInstance(); + invoke(instance, this, ""); + return instance; + }catch (Exception e){ + e.printStackTrace(); + } + + return null; + } + + + private void invoke(Object model, JcProperties properties, String prefix) throws + IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException { + + Class clz = model.getClass(); + Field[] fields = model.getClass().getDeclaredFields(); + for (Field field : fields) { + String type = field.getGenericType().toString(); + field.setAccessible(true); + + String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName())); + + prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix; + String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName(); + + + switch (type){ + case "class java.lang.String": + field.set(model, properties.getString(fillName)) ; + break; + case "byte": + field.setByte(model, Byte.valueOf(properties.getString(fillName))); + break; + case "short": + field.setShort(model, Short.valueOf(properties.getString(fillName))); + break; + case "int": + field.setInt(model, properties.getLong(fillName).intValue()) ; + break; + case "long": + field.setLong(model, properties.getLong(fillName)); + break; + case "double": + field.setDouble(model, Double.valueOf(properties.getString(fillName))); + break; + case "float": + field.setFloat(model, Float.valueOf(properties.getString(fillName))); + break; + case "boolean": + field.setBoolean(model, Boolean.parseBoolean(properties.getString(fillName))); + break; + case "class java.util.Date": + Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName)); + field.set(model,date) ; + break; + default: + String tn = field.getType().getTypeName(); + if("java.util.List".equals(tn)){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toList(val, String.class)); + }else if("java.util.Map".equals(tn)){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toMap(val)); + }else if(field.getType().isArray()){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toArray(val)); + }else{ + String[] ar = type.split(" "); + Object tinyObj = Class.forName(ar[1]).newInstance(); + invoke(tinyObj, properties, prefix); + field.set(model,tinyObj); + } + } + } + + } + +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java new file mode 100644 index 0000000..3a5d521 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java @@ -0,0 +1,142 @@ +package com.jd.platform.jlog.core; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.utils.StringUtil; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; +import java.util.*; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName Node.java 包装类 用于以后的支持key->node->[listener,properties] + * @createTime 2022年03月02日 20:37:00 + */ +public class NodeBak implements Serializable { + + + private String name; + + private ConfigChangeListener listener; + + private Properties properties; + + private String type; + + private Object val; + + + static Properties newPro = new Properties(); + + + public static void main(String[] args) throws IOException { + + String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; + // newPro.load(); + read(path); + + System.out.println(JSON.toJSONString(newPro)); + } + + protected static Map toMap(Properties properties) { + Map result = new HashMap<>(10); + Enumeration propertyNames = properties.propertyNames(); + while (propertyNames.hasMoreElements()) { + String name = (String) propertyNames.nextElement(); + String value = properties.getProperty(name); + result.put(name, value); + } + return result; + } + + + + + public static void read(String path) { + + FileReader fr = null; + LineNumberReader lnr = null; + String str; + + try { + fr = new FileReader(path); + lnr = new LineNumberReader(fr); + + Stack stack = new Stack<>(); + HashMap tinyMap = new LinkedHashMap<>(); + List tinyList = new ArrayList<>(); + String temKey = ""; + while ((str = lnr.readLine()) != null) { + if(StringUtil.isBlank(str) || !str.contains("=")){ + continue; + } + String[] lineArr = str.split("="); + String key = lineArr[0]; + String val = lineArr[1]; + + if(!key.contains(".")){ + newPro.put(key,val.trim()); + continue; + } + + String[] keyArr = key.split("\\."); + + String lastTinyKey = keyArr[keyArr.length - 1]; + if(lastTinyKey.contains("[") & lastTinyKey.contains("]")){ + String[] lastArr = lastTinyKey.split("\\["); + for (int i = 0; i < keyArr.length - 1; i++) { + stack.push(keyArr[i]); + } + stack.push(lastArr[0]); + + /* if(!temKey.equals(lastArr[0])){ + temKey = lastArr[0]; + String out = stack.pop(); + stack.push(temKey); + }*/ + //list + tinyList.add(val); + newPro.put(keyArr[0],tinyList); + }else{ + for (int i = 0; i < keyArr.length - 1; i++) { + if(stack.empty() || !stack.peek().equals(keyArr[i])){ + stack.push(keyArr[i]); + } + } + + /* if(!temKey.equals(lastTinyKey)){ + temKey = lastTinyKey; + String out = stack.pop(); + System.out.println("out==> "+ out +" temKey==> '"+temKey ); + stack.push(temKey); + }*/ + // map + tinyMap.put(keyArr[1],val.trim()); + newPro.put(keyArr[0],tinyMap); + } + System.out.println(" -- stack==> " + JSON.toJSONString(stack) ); + + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (fr != null) { + try { + fr.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (lnr != null) { + try { + lnr.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + } + +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java new file mode 100644 index 0000000..346f6b9 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java @@ -0,0 +1,45 @@ +package com.jd.platform.jlog.core; + +import java.util.List; +import java.util.Map; + +public class ObjObj{ + private int last; + private String max; + + private List ids; + + private Map ms; + + public int getLast() { + return last; + } + + public void setLast(int last) { + this.last = last; + } + + public String getMax() { + return max; + } + + public void setMax(String max) { + this.max = max; + } + + public List getIds() { + return ids; + } + + public void setIds(List ids) { + this.ids = ids; + } + + public Map getMs() { + return ms; + } + + public void setMs(Map ms) { + this.ms = ms; + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java new file mode 100644 index 0000000..a958816 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java @@ -0,0 +1,32 @@ +package com.jd.platform.jlog.core; + +public class Server{ + private int port; + private String addr; + private ObjObj objObj; + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getAddr() { + return addr; + } + + public void setAddr(String addr) { + this.addr = addr; + } + + + public ObjObj getObjObj() { + return objObj; + } + + public void setObjObj(ObjObj objObj) { + this.objObj = objObj; + } +} \ No newline at end of file diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java new file mode 100644 index 0000000..c8489f4 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java @@ -0,0 +1,39 @@ +package com.jd.platform.jlog.core; + +import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.tag.TagHandler; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName TagHandlerBuilder.java + * @Description TODO + * @createTime 2022年03月05日 22:07:00 + */ +public class TagHandlerBuilder { + + + + public static void buildTagHandler(TagConfig tagConfig, Configurator configurator){ + if(tagConfig == null){ + tagConfig = buildTagConfigByConfigurator(configurator); + } + TagHandler.buildHandler(tagConfig); + } + + + public static void refresh(){ + Configurator configurator = ConfiguratorFactory.getInstance(); + if(configurator == null){ + throw new RuntimeException("configurator is null"); + } + TagHandler.refresh(buildTagConfigByConfigurator(configurator)); + } + + + + + private static TagConfig buildTagConfigByConfigurator(Configurator configurator){ + return configurator.getObject("tag-config", TagConfig.class); + } +} diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index cc2a5c1..060fe06 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -1,6 +1,8 @@ package com.jd.platform.jlog.etcd; +import java.io.IOException; +import java.io.StringReader; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -39,7 +41,7 @@ public class EtcdConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; - private static final String PROPERTIES_PATH = "/jLog/properties"; + private static final String PROPERTIES_PATH = "/jLog/jLog.properties"; private static Properties PROPERTIES = new Properties(); @@ -56,7 +58,11 @@ public class EtcdConfigurator implements Configurator { } String val = keyValues.get(0).getValue().toStringUtf8(); if(StringUtil.isNotBlank(val)){ - PROPERTIES.putAll((Map)JSON.parse(val)); + try { + PROPERTIES.load(new StringReader(val)); + } catch (IOException e) { + e.printStackTrace(); + } } } @@ -73,6 +79,26 @@ public class EtcdConfigurator implements Configurator { } + @Override + public String getString(String key) { + return null; + } + + @Override + public Long getLong(String key) { + return null; + } + + @Override + public List getList(String key) { + return null; + } + + @Override + public T getObject(String key, Class clz) { + return null; + } + @Override public String getConfig(String key) { Object val = PROPERTIES.get(key); diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index 6524c83..53bf0e9 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -5,6 +5,7 @@ import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.EtcdClient; import com.ibm.etcd.client.kv.KvClient; +import com.ibm.etcd.client.kv.WatchUpdate; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.ConfigChangeType; @@ -30,19 +31,24 @@ public class EtcdListener implements ConfigChangeListener { public EtcdListener(String node) { - iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(node)).start(); - System.out.println("构造器EtcdListener"); + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(node)).asPrefix().start(); + System.out.println("构造器EtcdListener"+node); getExecutorService().submit(() -> { while (iterator.hasNext()){ - Event eve = iterator.next().getEvents().get(0); - KeyValue kv = eve.getKv(); - Event.EventType eveType = eve.getType(); - ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; + try { + WatchUpdate update = iterator.next(); + Event eve = update.getEvents().get(0); + KeyValue kv = eve.getKv(); + Event.EventType eveType = eve.getType(); + ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; + ConfigChangeEvent event = new ConfigChangeEvent(); + event.setKey(node).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + onChangeEvent(event); + }catch (RuntimeException e){ + e.printStackTrace(); + } - ConfigChangeEvent event = new ConfigChangeEvent(); - event.setKey(node).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); - onChangeEvent(event); } }); } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java index 678d071..88212aa 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -20,7 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; -import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; +import static com.jd.platform.jlog.core.Constant.*; import static com.jd.platform.jlog.nacos.NacosConstant.*; @@ -78,6 +78,26 @@ public class NacosConfigurator implements Configurator { } + @Override + public String getString(String key) { + return null; + } + + @Override + public Long getLong(String key) { + return null; + } + + @Override + public List getList(String key) { + return null; + } + + @Override + public T getObject(String key, Class clz) { + return null; + } + @Override public String getConfig(String key) { return getConfig(key, DEFAULT_TIMEOUT); @@ -198,9 +218,18 @@ public class NacosConfigurator implements Configurator { private static Properties getConfigProperties() { Properties properties = new Properties(); - String address = FILE_CONFIG.getConfig(PRO_SERVER_ADDR_KEY); + String address = FILE_CONFIG.getConfig(SERVER_ADDR_KEY); if (address != null) { - properties.setProperty(PRO_SERVER_ADDR_KEY, address); + properties.setProperty(SERVER_ADDR_KEY, address); + } + + String namespace = FILE_CONFIG.getConfig(NAMESPACE_KEY); + if (namespace != null) { + properties.setProperty(NAMESPACE_KEY, namespace); + }else{ + if (System.getProperty(NAMESPACE_KEY) != null) { + properties.setProperty(NAMESPACE_KEY, System.getProperty(NAMESPACE_KEY)); + } } return properties; } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java index 326932c..513671f 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java @@ -12,5 +12,4 @@ public class NacosConstant { static final String JLOG_GROUP = "JLOG_GROUP"; static final String DEFAULT_DATA_ID = "jLog.properties"; - static final String PRO_SERVER_ADDR_KEY = "serverAddr"; } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java index 68e3218..a67bced 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java @@ -28,11 +28,11 @@ public class NacosTest { NamingService naming = NamingFactory.createNamingService(serverAddr); naming.registerInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1"); Thread.sleep(3000); - NacosListener server = new NacosListener(DEFAULT_DATA_ID); + NacosListener server = new NacosListener(); naming.subscribe("nacos.test1", JLOG_GROUP,server); String content = configService.getConfig(dataId, group, 2000L); System.out.println("content: "+content); - NacosListener nL = new NacosListener(DEFAULT_DATA_ID); + NacosListener nL = new NacosListener(); configService.addListener(dataId, group, nL); System.out.println("新增完成"); diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java index bea239a..c1130bb 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -39,10 +39,9 @@ public class ZkConfigurator implements Configurator { public ZkConfigurator() throws Exception { - System.out.println("### SERVER_ADDR_KEY ===> "+FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); if (zkClient == null) { synchronized (ZkConfigurator.class) { - zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getConfig(SERVER_ADDR_KEY)) + zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getString(SERVER_ADDR_KEY)) // 连接超时时间 .sessionTimeoutMs(6000) // 会话超时时间 @@ -65,30 +64,24 @@ public class ZkConfigurator implements Configurator { @Override - public String getConfig(String key) { - return getConfig(key, DEFAULT_TIMEOUT); + public String getString(String key) { + return getString(key); } - @Override - public String getConfig(String key, long timeoutMills) { - String value = PROPERTIES.getProperty(key); - if (value != null) { - return value; - } + public Long getLong(String key) { + return null; + } - value = System.getProperty(key); - if (value != null) { - return value; - } - try { - loadZkData(); - } catch (Exception e) { - return null; - } - return PROPERTIES.getProperty(key); + @Override + public List getList(String key) { + return null; } + @Override + public T getObject(String key, Class clz) { + return null; + } @Override @@ -140,8 +133,11 @@ public class ZkConfigurator implements Configurator { throw new RuntimeException("no support"); } LOGGER.info("ZK添加监听器, node:{}", node); - ZKLISTENER = new ZkListener(node); - ZKLISTENER.onProcessEvent(new ConfigChangeEvent()); + if(ZKLISTENER == null){ + synchronized (ZkConfigurator.class){ + ZKLISTENER = new ZkListener(node); + } + } } @@ -159,7 +155,7 @@ public class ZkConfigurator implements Configurator { @Override public List getConfigByPrefix(String prefix) { try { - String val = getConfig(prefix); + String val = getString(prefix); return FastJsonUtils.toList(val,String.class); } catch (Exception e) { e.printStackTrace(); diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java index 388b666..9877341 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -32,7 +32,6 @@ public class ZkListener implements ConfigChangeListener { public ZkListener(String path) { this.path = path; cache = new NodeCache(zkClient, path); - LOGGER.info("构造ZkListener:{}",path); try { cache.start(true); } catch (Exception e) { @@ -43,9 +42,9 @@ public class ZkListener implements ConfigChangeListener { if(null!=cache.getCurrentData()){ value = new String(cache.getCurrentData().getData()); } + onChangeEvent(null); System.out.println("=####====== "+value); }); - System.out.println("=####= SIZE===== "+cache.getListenable().size()); } @@ -62,19 +61,23 @@ public class ZkListener implements ConfigChangeListener { @Override public void onChangeEvent(ConfigChangeEvent event) { - LOGGER.info("ZK数据变更-当前监听器关注的path:{}", path); - Properties propsTmp = new Properties(PROPERTIES); + LOGGER.info("ZK数据变更-当前监听器关注的path:{} PROPERTIES:{}", path, JSON.toJSONString(PROPERTIES)); + Properties props = new Properties(); + props.putAll(PROPERTIES); try { - LOGGER.info("ZK数据变更,旧Properties:{}", JSON.toJSONString(propsTmp)); + LOGGER.info("ZK数据变更,旧Properties:{}", JSON.toJSONString(props)); loadZkData(); LOGGER.info("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES)); } catch (Exception e) { e.printStackTrace(); } - Set diffKeys = CollectionUtil.diffKeys(propsTmp, PROPERTIES); + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); if(!diffKeys.isEmpty()){ for (String diffKey : diffKeys) { - LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey, new ConfigChangeEvent(diffKey, propsTmp.get(diffKey).toString(), PROPERTIES.get(diffKey).toString())); + LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey, + new ConfigChangeEvent(diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey).toString()))); } } } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java index 15039e7..f66b9ec 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java @@ -3,13 +3,10 @@ package com.jd.platform.jlog.zk; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.recipes.cache.NodeCache; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; import org.apache.curator.retry.ExponentialBackoffRetry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.jd.platform.jlog.zk.ZkConstant.SERVER_ADDR_KEY; - /** * @author tangbohu * @version 1.0.0 diff --git a/clientdemo/pom.xml b/example/pom.xml similarity index 96% rename from clientdemo/pom.xml rename to example/pom.xml index 09a2c83..b177a8e 100644 --- a/clientdemo/pom.xml +++ b/example/pom.xml @@ -10,10 +10,10 @@ com.jd.platfrom.jlog - clientdemo + example 0.0.1-SNAPSHOT - clientdemo - Demo project for using JLog + example + example for using JLog 1.8 diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java similarity index 79% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java index 8da0ed4..fec83ce 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java @@ -11,11 +11,11 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; * @date 2021-12-27 */ @SpringBootApplication -public class ClientDemoApplication { +public class ExampleApplication { public static void main(String[] args) { try { - SpringApplication.run(ClientDemoApplication.class, args); + SpringApplication.run(ExampleApplication.class, args); }catch (Exception e){ e.printStackTrace(); } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java similarity index 98% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java index 6db352c..a450351 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -30,9 +30,9 @@ public class TracerLogbackAppender extends AppenderBase { protected void append(ILoggingEvent iLoggingEvent) { try { long tracerId = TracerHolder.getTracerId(); - /* if (0L == tracerId) { + if (0L == tracerId) { return; - }*/ + } RunLogMessage logMessage = getLogMessage(iLoggingEvent); UdpSender.offerLogger(logMessage); } catch (Exception e) { @@ -62,7 +62,7 @@ public class TracerLogbackAppender extends AppenderBase { String formattedMessage = getMessage(loggingEvent); logMessage.setContent(formattedMessage); Map map = TagHandler.extractLogTag(formattedMessage); - + logMessage.setTagMap(map); return logMessage; } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java similarity index 100% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java similarity index 56% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 2fbd065..8533e0c 100644 --- a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -1,25 +1,38 @@ package com.jd.platform.jlog.clientdemo.web; +import com.jd.platform.jlog.clientdemo.config.DemoConfig; import com.jd.platform.jlog.common.model.TracerBean; +import com.jd.platform.jlog.common.tag.TagConfig; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; /** * @author shenkaiwen5 * @version 1.0 * @date 2021-12-27 */ +@Component +//@ConfigurationProperties() @RestController public class TestController { + private TagConfig tagConfig ; + + public TagConfig getTagConfig() { + return tagConfig; + } + + public void setTagConfig(TagConfig tagConfig) { + this.tagConfig = tagConfig; + } + /** * do nothing * just as an adapter for this project common log helper @@ -39,7 +52,7 @@ public class TestController { e.printStackTrace(); } - String val = configurator.getConfig("/test"); + String val = configurator.getString("/test"); System.out.println("val ===> "+val); RequestLog.info("哈哈哈哈哈哈"); @@ -48,14 +61,16 @@ public class TestController { @RequestMapping("/log") public Object log() { - RequestLog.info("|tag3=val3||tag4=val4||这是随便的log|"); + RequestLog.info("|errno=val3||node=val4||这是随便的log|"); return 1; } @PostMapping(value = "/test", consumes = MediaType.APPLICATION_JSON_VALUE) - public Object test(@RequestBody TestReq req) { - + public Object test(@RequestParam Integer uid, @RequestParam Integer newKey,@RequestBody TestReq req) { + String config = ConfiguratorFactory.getInstance().getString("reqTags"); + // System.out.println("tagConfig ===> " + tagConfig.toString()); + RequestLog.info("|errno=val3||node=val4||这是随便的log|"); return 1; } diff --git a/clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java similarity index 100% rename from clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java rename to example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties new file mode 100644 index 0000000..9c7c1c6 --- /dev/null +++ b/example/src/main/resources/application.properties @@ -0,0 +1,16 @@ +server.port=8085 + +tag-config.delimiter="|" + +tag-config.logConfig.limit=99 + +tag-config.logConfig.cfg[0]="AA" +tag-config.logConfig.cfg[1]="BB" +tag-config.logConfig.aa[0]="CC" +tag-config.logConfig.aa[1]="DD" + + +tag-config.reqTags[0]="AA" +tag-config.reqTags[1]="BB" +tag-config.logTags[0]="CC" +tag-config.logTags[1]="XSSX" diff --git a/clientdemo/src/main/resources/application.ymlbak b/example/src/main/resources/application.ymlbak similarity index 100% rename from clientdemo/src/main/resources/application.ymlbak rename to example/src/main/resources/application.ymlbak diff --git a/clientdemo/src/main/resources/application.yml b/example/src/main/resources/bakapplication.yml similarity index 73% rename from clientdemo/src/main/resources/application.yml rename to example/src/main/resources/bakapplication.yml index faa4d69..bce0acf 100644 --- a/clientdemo/src/main/resources/application.yml +++ b/example/src/main/resources/bakapplication.yml @@ -1,7 +1,9 @@ -testKey: 111 -serverAddr: 101.42.242.201:2181 +serverAddr: http://101.42.242.201:8848 server: port: 8085 + +test: true + workers: - 111 - 222 @@ -11,6 +13,7 @@ tag-config: reqTags: - uid - ip + - newK logTags: - errno - node diff --git a/example/src/main/resources/bakjLog.properties b/example/src/main/resources/bakjLog.properties new file mode 100644 index 0000000..4f16091 --- /dev/null +++ b/example/src/main/resources/bakjLog.properties @@ -0,0 +1,6 @@ +server.port=8085 +tag-config.reqTags[0]="AA" +tag-config.reqTags[1]="BB" +tag-config.logTags[0]="CC" +tag-config.logTags[1]="DD" +tag-config.delimiter="|" diff --git a/clientdemo/src/main/resources/logback-spring.xml b/example/src/main/resources/logback-spring.xml similarity index 100% rename from clientdemo/src/main/resources/logback-spring.xml rename to example/src/main/resources/logback-spring.xml diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java b/example/src/test/java/com/jd/platform/jlog/test/Common.java similarity index 93% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java rename to example/src/test/java/com/jd/platform/jlog/test/Common.java index 37242b1..3db0d8e 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/Common.java +++ b/example/src/test/java/com/jd/platform/jlog/test/Common.java @@ -24,9 +24,9 @@ public class Common { public static void getTest(Configurator configurator){ LOGGER.info("配置器类型：{}", configurator.getType()); - String addr = configurator.getConfig("serverAddr"); + String addr = configurator.getString("serverAddr"); LOGGER.info("配置器get addr：{}", addr); - String reqTags = configurator.getConfig("reqTags"); + String reqTags = configurator.getString("reqTags"); LOGGER.info("配置器get reqTags：{}", reqTags); List workers = configurator.getConfigByPrefix("workers"); LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); diff --git a/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java new file mode 100644 index 0000000..2945261 --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java @@ -0,0 +1,91 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName EtcdConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月03日 07:35:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class EtcdConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(EtcdConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + // @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getConfigByPrefix("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.1.1.0"; + if(workers.contains(myIp)){ + // do nothing + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getConfigByPrefix("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getString("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + configurator.addConfigListener("/jLog"); + LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); + configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(5000); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + Thread.sleep(5000); + configurator.removeConfigListener("/jLog"); + Thread.sleep(5000); + // LOGGER.info("移除监听器后:修改配置testKey = {}",i2); + // configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + + + configurator.addConfigListener("/jLog"); + LOGGER.info("第二次添加监听器"); + Thread.sleep(22000); + + } +} diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java similarity index 63% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java rename to example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java index 234c90b..b8ec6ec 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -1,7 +1,7 @@ package com.jd.platform.jlog.test; import com.alibaba.fastjson.JSON; -import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.clientdemo.ExampleApplication; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.junit.Before; @@ -14,10 +14,8 @@ import org.springframework.test.context.junit4.SpringRunner; import org.yaml.snakeyaml.Yaml; import java.io.*; -import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Random; import static com.jd.platform.jlog.test.Common.getTest; import static com.jd.platform.jlog.test.Common.modifyFile; @@ -30,7 +28,7 @@ import static com.jd.platform.jlog.test.Common.modifyFile; * @Description TODO * @createTime 2022年02月28日 19:45:00 */ -@SpringBootTest(classes = ClientDemoApplication.class) +@SpringBootTest(classes = ExampleApplication.class) @RunWith(SpringRunner.class) public class FileConfiguratorTest { @@ -52,8 +50,8 @@ public class FileConfiguratorTest { @Test public void testAddConfigListener() throws Exception { - configurator.addConfigListener("/application.yml"); - String path = "/Users/didi/Desktop/jlog/clientdemo/target/classes/application.yml"; + configurator.addConfigListener("/bakapplication.yml"); + String path = "/Users/didi/Desktop/jlog/example/target/classes/bakapplication.yml"; Properties props = new Properties(); FileInputStream fis = new FileInputStream(new File(path)); if (path.contains("yml")) { @@ -61,15 +59,25 @@ public class FileConfiguratorTest { } else { props.load(fis); } - LOGGER.info("读取文件：{}最新配置:{}", path, JSON.toJSONString(props)); + LOGGER.info("读取文件：{} 最新配置:{}", path, JSON.toJSONString(props)); modifyFile(path); LOGGER.info("修改文件完毕 准备触发监听器"); - Thread.sleep(1000); - LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getConfig("testKey")); - LOGGER.info("移除监听器"); - configurator.removeConfigListener("/application.yml"); + Thread.sleep(2000); + LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getString("testKey")); + Thread.sleep(12000); + LOGGER.info("移除监听器之前，testKey：{}",configurator.getString("testKey")); + configurator.removeConfigListener("/bakapplication.yml"); + LOGGER.info("移除监听器之后，testKey：{}",configurator.getString("testKey")); modifyFile(path); - LOGGER.info("修改文件完毕 准备验证监听器是否停止 最新testKey={}", configurator.getConfig("testKey")); + LOGGER.info("修改文件完毕 准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + + + LOGGER.info("再次添加监听器"); + configurator.addConfigListener("/bakapplication.yml"); + modifyFile(path); + LOGGER.info("修改文件完毕"); + Thread.sleep(12000); + } diff --git a/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java new file mode 100644 index 0000000..bcc1be6 --- /dev/null +++ b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java @@ -0,0 +1,87 @@ +package com.jd.platform.jlog.test; + +import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.clientdemo.ExampleApplication; +import com.jd.platform.jlog.core.Configurator; +import com.jd.platform.jlog.core.ConfiguratorFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Random; + +import static com.jd.platform.jlog.test.Common.getTest; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName NacosConfiguratorTest.java + * @Description TODO + * @createTime 2022年03月01日 07:35:00 + */ +@SpringBootTest(classes = ExampleApplication.class) +@RunWith(SpringRunner.class) +public class NacosConfiguratorTest { + + + + private final static Logger LOGGER = LoggerFactory.getLogger(NacosConfiguratorTest.class); + + + private Configurator configurator = null; + + + + @Before + public void init() { + configurator = ConfiguratorFactory.getInstance(); + getTest(configurator); + } + + @Test + public void testUpdateCFG() throws Exception { + List workers = configurator.getConfigByPrefix("workers"); + LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); + String myIp = "121.1.1.0"; + if(workers.contains(myIp)){ + // do nothing + LOGGER.info("自己的IP还在配置list里 什么也不做"); + return; + }else{ + LOGGER.info("自己的IP不在配置list里 添加进去并发布"); + workers.add(myIp); + } + configurator.putConfig("workers",JSON.toJSONString(workers)); + List workers2 = configurator.getConfigByPrefix("workers"); + LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); + } + + @Test + public void testAddConfigListener() throws Exception { + int i1 = new Random().nextInt(2000); + int i2 = new Random().nextInt(2000); + + String val1 = configurator.getString("testKey"); + LOGGER.info("初始化的testKey的val:{}", val1); + configurator.addConfigListener("bakjLog.properties"); + LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); + // configurator.putConfig("testKey",i1 + ""); + LOGGER.info("修改完毕 准备触发监听器"); + Thread.sleep(1000); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + Thread.sleep(1000); + configurator.removeConfigListener("bakjLog.properties"); + Thread.sleep(1000); + LOGGER.info("移除监听器后:修改配置testKey = {}",i2); + configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + + Thread.sleep(35000); + } +} diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java b/example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java similarity index 95% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java rename to example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java index 140b2d5..0ef5a23 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java @@ -2,7 +2,7 @@ package com.jd.platform.jlog.test; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.clientdemo.ExampleApplication; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IpUtils; import com.jd.platform.jlog.common.utils.ZstdUtils; @@ -18,7 +18,7 @@ import java.util.*; * @version 1.0 * @date 2021-12-27 */ -//@SpringBootTest(classes = ClientDemoApplication.class) +//@SpringBootTest(classes = ExampleApplication.class) //@RunWith(SpringRunner.class) public class TracerPacketTest { diff --git a/clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java similarity index 69% rename from clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java rename to example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java index 5850d22..4f320dd 100644 --- a/clientdemo/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java @@ -1,7 +1,7 @@ package com.jd.platform.jlog.test; import com.alibaba.fastjson.JSON; -import com.jd.platform.jlog.clientdemo.ClientDemoApplication; +import com.jd.platform.jlog.clientdemo.ExampleApplication; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.junit.Before; @@ -11,17 +11,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; -import org.yaml.snakeyaml.Yaml; -import java.io.File; -import java.io.FileInputStream; import java.util.List; -import java.util.Map; -import java.util.Properties; + import java.util.Random; import static com.jd.platform.jlog.test.Common.getTest; -import static com.jd.platform.jlog.test.Common.modifyFile; /** * @author tangbohu @@ -30,7 +25,7 @@ import static com.jd.platform.jlog.test.Common.modifyFile; * @Description TODO * @createTime 2022年03月01日 07:35:00 */ -@SpringBootTest(classes = ClientDemoApplication.class) +@SpringBootTest(classes = ExampleApplication.class) @RunWith(SpringRunner.class) public class ZKConfiguratorTest { @@ -53,7 +48,7 @@ public class ZKConfiguratorTest { public void testUpdateCFG() throws Exception { List workers = configurator.getConfigByPrefix("workers"); LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); - String myIp = "121.1.1.1"; + String myIp = "121.1.1.0"; if(workers.contains(myIp)){ // do nothing LOGGER.info("自己的IP还在配置list里 什么也不做"); @@ -72,20 +67,22 @@ public class ZKConfiguratorTest { int i1 = new Random().nextInt(2000); int i2 = new Random().nextInt(2000); - String val1 = configurator.getConfig("testKey"); + String val1 = configurator.getString("testKey"); LOGGER.info("初始化的testKey的val:{}", val1); - configurator.addConfigListener("/application.yml"); - LOGGER.info("添加监听器:随机数：{}", i1); - configurator.putConfig("testKey",i1 + ""); + configurator.addConfigListener("/bakjLog.properties"); + LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); + // configurator.putConfig("testKey",i1 + ""); LOGGER.info("修改完毕 准备触发监听器"); Thread.sleep(1000); - String val2 = configurator.getConfig("testKey"); - LOGGER.info("修改后的的val:{}", val2); - LOGGER.info("移除监听器:随机数：{}",i2); - Thread.sleep(3000); - configurator.removeConfigListener("/application.yml"); + String val2 = configurator.getString("testKey"); + LOGGER.info("第一次修改后的的val:{}", val2); + Thread.sleep(1000); + configurator.removeConfigListener("/bakjLog.properties"); Thread.sleep(1000); - configurator.putConfig("testKey",i2 + ""); - LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getConfig("testKey")); + LOGGER.info("移除监听器后:修改配置testKey = {}",i2); + // configurator.putConfig("testKey",i2 + ""); + LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); + + Thread.sleep(35000); } } diff --git a/pom.xml b/pom.xml index eb3fca9..4c92e3e 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ clientlog4j2 clientlogback Dashboard - clientdemo + example config -- Gitee From 01084112e32b19e28178a8fe156ded8d5cf05dbb Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Thu, 10 Mar 2022 21:15:53 +0800 Subject: [PATCH 16/22] =?UTF-8?q?=E5=8C=BA=E5=88=86=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/client/TracerClientStarter.java | 2 +- .../jlog/client/filter/HttpFilter.java | 2 +- .../common/{tag => handler}/CollectMode.java | 2 +- .../jlog/common/handler/CompressHandler.java | 11 +++ .../jlog/common/{tag => handler}/Tag.java | 2 +- .../common/{tag => handler}/TagConfig.java | 83 ++++--------------- .../common/{tag => handler}/TagHandler.java | 41 +++++++-- .../com/jd/platform/jlog/core/FileNode.java | 9 +- .../jd/platform/jlog/core/JcProperties.java | 9 +- .../platform/jlog/core/TagHandlerBuilder.java | 6 +- .../appender/TracerLogbackAppender.java | 3 +- .../jlog/clientdemo/config/DemoConfig.java | 2 +- .../jlog/clientdemo/web/TestController.java | 4 +- .../src/main/resources/application.properties | 20 ++--- example/src/main/resources/bakjLog.properties | 11 ++- .../jlog/worker/config/ConfigCenter.java | 2 +- 16 files changed, 94 insertions(+), 115 deletions(-) rename common/src/main/java/com/jd/platform/jlog/common/{tag => handler}/CollectMode.java (98%) create mode 100644 common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java rename common/src/main/java/com/jd/platform/jlog/common/{tag => handler}/Tag.java (89%) rename common/src/main/java/com/jd/platform/jlog/common/{tag => handler}/TagConfig.java (54%) rename common/src/main/java/com/jd/platform/jlog/common/{tag => handler}/TagHandler.java (85%) diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index 1f72fb1..a4d5c04 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -4,7 +4,7 @@ package com.jd.platform.jlog.client; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.task.Monitor; -import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import com.jd.platform.jlog.core.TagHandlerBuilder; diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 0da8bf6..f433094 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -5,7 +5,7 @@ import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.tag.TagHandler; +import com.jd.platform.jlog.common.handler.TagHandler; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java similarity index 98% rename from common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java index 191ece5..d05336f 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/CollectMode.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.common.tag; +package com.jd.platform.jlog.common.handler; /** * @author tangbohu diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java new file mode 100644 index 0000000..02b0bfe --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -0,0 +1,11 @@ +package com.jd.platform.jlog.common.handler; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName CompressHandler.java + * @Description TODO + * @createTime 2022年03月10日 20:52:00 + */ +public class CompressHandler { +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java b/common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java similarity index 89% rename from common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java index 72fc759..979bea3 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/Tag.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.common.tag; +package com.jd.platform.jlog.common.handler; import java.lang.annotation.*; diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java similarity index 54% rename from common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java index c6e53a7..1e1507c 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagConfig.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java @@ -1,10 +1,10 @@ -package com.jd.platform.jlog.common.tag; +package com.jd.platform.jlog.common.handler; import java.io.Serializable; import java.util.List; -import static com.jd.platform.jlog.common.tag.CollectMode.COMPRESS_ALL; -import static com.jd.platform.jlog.common.tag.CollectMode.EXTRACT_ALL; +import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_ALL; +import static com.jd.platform.jlog.common.handler.CollectMode.EXTRACT_ALL; /** * @author tangbohu @@ -24,6 +24,11 @@ public class TagConfig implements Serializable { */ private List logTags; + /** + * 返参的tag + */ + private List respTags; + /** * 自定义正则 可为空 */ @@ -66,6 +71,14 @@ public class TagConfig implements Serializable { this.logTags = logTags; } + public List getRespTags() { + return respTags; + } + + public void setRespTags(List respTags) { + this.respTags = respTags; + } + public String getRegex() { return regex; } @@ -106,75 +119,13 @@ public class TagConfig implements Serializable { this.compress = compress; } - public static final class Builder { - private List reqTags; - private List logTags; - private String regex; - private String delimiter; - private String join; - private long extract; - private long compress; - - public Builder() { - } - - public static Builder aTagConfig() { - return new Builder(); - } - - public Builder reqTags(List reqTags) { - this.reqTags = reqTags; - return this; - } - - public Builder logTags(List logTags) { - this.logTags = logTags; - return this; - } - - public Builder regex(String regex) { - this.regex = regex; - return this; - } - - public Builder delimiter(String delimiter) { - this.delimiter = delimiter; - return this; - } - - public Builder join(String join) { - this.join = join; - return this; - } - - public Builder extract(long extract) { - this.extract = extract; - return this; - } - - public Builder compress(long compress) { - this.compress = compress; - return this; - } - - public TagConfig build() { - TagConfig tagConfig = new TagConfig(); - tagConfig.setReqTags(reqTags); - tagConfig.setLogTags(logTags); - tagConfig.setRegex(regex); - tagConfig.setDelimiter(delimiter); - tagConfig.setJoin(join); - tagConfig.setExtract(extract); - tagConfig.setCompress(compress); - return tagConfig; - } - } @Override public String toString() { return "TagConfig{" + "reqTags=" + reqTags + ", logTags=" + logTags + + ", respTags=" + respTags + ", regex='" + regex + '\'' + ", delimiter='" + delimiter + '\'' + ", join='" + join + '\'' + diff --git a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java similarity index 85% rename from common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java index 831a7bf..404fdce 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/tag/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.common.tag; +package com.jd.platform.jlog.common.handler; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.utils.CollectionUtil; @@ -15,9 +15,9 @@ import java.util.regex.Pattern; import static com.jd.platform.jlog.common.constant.Constant.EXTRACT_MIN_LEN; import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY; import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY_MAX_LEN; -import static com.jd.platform.jlog.common.tag.CollectMode.*; -import static com.jd.platform.jlog.common.tag.CollectMode.E_LOG; -import static com.jd.platform.jlog.common.tag.CollectMode.E_REQ; +import static com.jd.platform.jlog.common.handler.CollectMode.*; +import static com.jd.platform.jlog.common.handler.CollectMode.E_LOG; +import static com.jd.platform.jlog.common.handler.CollectMode.E_REQ; import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; /** @@ -34,6 +34,8 @@ public class TagHandler { private Set logTags; + private Set respTags; + private String delimiter = "|"; private int delimiterLen = delimiter.length(); @@ -44,8 +46,6 @@ public class TagHandler { private long extract; - private long compress; - private static volatile TagHandler INSTANCE = null; /** @@ -60,9 +60,9 @@ public class TagHandler { TagHandler handler = new TagHandler(); handler.extract = tagConfig.getExtract(); - handler.compress = tagConfig.getCompress(); handler.reqTags = new HashSet<>(tagConfig.getReqTags()); handler.logTags = new HashSet<>(tagConfig.getLogTags()); + handler.respTags = new HashSet<>(tagConfig.getRespTags()); String regex = tagConfig.getRegex(); if(StringUtil.isNotEmpty(regex)){ @@ -110,7 +110,7 @@ public class TagHandler { /** - * 提取普通日志中的标签 + * 提取普通log里标签 * @param content 内容 * @return tags */ @@ -141,6 +141,30 @@ public class TagHandler { } + /** + * 提取返回结果的tag + * @param resp 参数 + * @return map + */ + public static Map extractRespTag(Map resp) { + + if(INSTANCE == null || !isMatched(INSTANCE.extract, E_REQ)){ return null; } + + System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(INSTANCE.reqTags)); + + Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); + for (String tag : INSTANCE.reqTags) { + Object val = resp.get(tag); + if(val != null){ + requestMap.put(tag, val); + } + } + System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(requestMap)); + return requestMap; + } + + + /** * 刷新标签处理器 加锁是为了防止极端情况下, 先到的config1覆盖后到的config2 @@ -162,7 +186,6 @@ public class TagHandler { ", join='" + join + '\'' + ", pattern=" + pattern + ", extract=" + extract + - ", compress=" + compress + '}'; } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java index 49634af..30fef46 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -1,16 +1,15 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.CollectMode; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; /** * @author tangbohu * @version 1.0.0 * @ClassName FileNode.java - * @Description TODO * @createTime 2022年03月08日 16:32:00 */ public final class FileNode { @@ -20,6 +19,12 @@ public final class FileNode { private long lastModity; public static void main(String[] args) throws IOException { + + System.out.println(CollectMode.COMPRESS_ALL); + System.out.println(CollectMode.EXTRACT_ALL); + + + System.out.println(); JcProperties properties = new JcProperties(); String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; properties.load(new FileInputStream(path)); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java index 8bd02bd..9d2368e 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java @@ -5,12 +5,7 @@ import com.jd.platform.jlog.common.utils.ConfigUtil; import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.common.utils.StringUtil; -import java.beans.IntrospectionException; -import java.beans.PropertyDescriptor; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; @@ -30,11 +25,11 @@ public class JcProperties extends Properties { } public String getString(String key) { - Object val = super.get(key); + Object val = get(key); if(val == null){ return null; } - return String.valueOf(super.get(key)); + return String.valueOf(val); } public Long getLong(String key) { diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java index c8489f4..c3bde54 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java @@ -1,7 +1,7 @@ package com.jd.platform.jlog.core; -import com.jd.platform.jlog.common.tag.TagConfig; -import com.jd.platform.jlog.common.tag.TagHandler; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.common.handler.TagHandler; /** * @author tangbohu @@ -34,6 +34,6 @@ public class TagHandlerBuilder { private static TagConfig buildTagConfigByConfigurator(Configurator configurator){ - return configurator.getObject("tag-config", TagConfig.class); + return configurator.getObject("handler-config", TagConfig.class); } } diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java index a450351..70417b0 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -4,12 +4,11 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.ThrowableProxy; import ch.qos.logback.core.AppenderBase; -import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.log.LogExceptionStackTrace; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.model.RunLogMessage; -import com.jd.platform.jlog.common.tag.TagHandler; +import com.jd.platform.jlog.common.handler.TagHandler; import org.slf4j.helpers.MessageFormatter; import java.util.Map; diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java index 8fd2d29..00a7fe8 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java @@ -2,7 +2,7 @@ package com.jd.platform.jlog.clientdemo.config; import com.jd.platform.jlog.client.TracerClientStarter; import com.jd.platform.jlog.client.filter.HttpFilter; -import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.handler.TagConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 8533e0c..0a3b385 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -1,14 +1,12 @@ package com.jd.platform.jlog.clientdemo.web; -import com.jd.platform.jlog.clientdemo.config.DemoConfig; import com.jd.platform.jlog.common.model.TracerBean; -import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.*; diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties index 9c7c1c6..b924a69 100644 --- a/example/src/main/resources/application.properties +++ b/example/src/main/resources/application.properties @@ -1,16 +1,10 @@ server.port=8085 +tag-config.reqTags=["uid","url"] +tag-config.logTags=["node","bizType"] +tag-config.respTags=["errno","msg"] +tag-config.regex="" tag-config.delimiter="|" - -tag-config.logConfig.limit=99 - -tag-config.logConfig.cfg[0]="AA" -tag-config.logConfig.cfg[1]="BB" -tag-config.logConfig.aa[0]="CC" -tag-config.logConfig.aa[1]="DD" - - -tag-config.reqTags[0]="AA" -tag-config.reqTags[1]="BB" -tag-config.logTags[0]="CC" -tag-config.logTags[1]="XSSX" +tag-config.join="=" +tag-config.extract=41 +tag-config.compress=81 diff --git a/example/src/main/resources/bakjLog.properties b/example/src/main/resources/bakjLog.properties index 4f16091..ef21cc4 100644 --- a/example/src/main/resources/bakjLog.properties +++ b/example/src/main/resources/bakjLog.properties @@ -1,6 +1,9 @@ server.port=8085 -tag-config.reqTags[0]="AA" -tag-config.reqTags[1]="BB" -tag-config.logTags[0]="CC" -tag-config.logTags[1]="DD" +tag-config.reqTags=["uid","url"] +tag-config.logTags=["node","bizType"] +tag-config.respTags=["errno","msg"] +tag-config.regex="" tag-config.delimiter="|" +tag-config.join="=" +tag-config.extract=41 +tag-config.compress=81 \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java index 21d33b5..abae934 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java @@ -1,7 +1,7 @@ package com.jd.platform.jlog.worker.config; -import com.jd.platform.jlog.common.tag.TagConfig; +import com.jd.platform.jlog.common.handler.TagConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; -- Gitee From 172ab9fe6b9ca122300d1122bbe21da35487cbcd Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Fri, 11 Mar 2022 17:11:47 +0800 Subject: [PATCH 17/22] =?UTF-8?q?=E5=8C=BA=E5=88=86=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/client/TracerClientStarter.java | 4 +- .../jlog/client/filter/HttpFilter.java | 7 +-- .../jlog/common/handler/CollectMode.java | 4 +- .../jlog/common/handler/CompressHandler.java | 57 +++++++++++++++++++ .../jlog/common/handler/TagConfig.java | 13 ----- .../jlog/common/handler/TagHandler.java | 42 +++++++------- .../platform/jlog/core/FileConfigurator.java | 2 +- .../com/jd/platform/jlog/core/FileNode.java | 25 +++++++- ...andlerBuilder.java => HandlerBuilder.java} | 8 ++- .../jd/platform/jlog/core/JcProperties.java | 2 +- .../src/main/resources/application.properties | 5 +- 11 files changed, 117 insertions(+), 52 deletions(-) rename config/config-core/src/main/java/com/jd/platform/jlog/core/{TagHandlerBuilder.java => HandlerBuilder.java} (74%) diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index a4d5c04..9c329a0 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -7,7 +7,7 @@ import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; -import com.jd.platform.jlog.core.TagHandlerBuilder; +import com.jd.platform.jlog.core.HandlerBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +105,7 @@ public class TracerClientStarter { private void initJLogConfig(){ LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig)); Configurator configurator = ConfiguratorFactory.getInstance(); - TagHandlerBuilder.buildTagHandler(tagConfig, configurator); + HandlerBuilder.buildHandler(tagConfig, configurator); configurator.addConfigListener("/application.properties"); // configurator.addConfigListener("/application.yml"); diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index f433094..5756a80 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -5,6 +5,7 @@ import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; +import com.jd.platform.jlog.common.handler.CompressHandler; import com.jd.platform.jlog.common.handler.TagHandler; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IdWorker; @@ -128,12 +129,8 @@ public class HttpFilter implements Filter { filterChain.doFilter(servletRequest, mResp); byte[] contentBytes = mResp.getContent(); String content = new String(contentBytes); - //最终的要发往worker的response，经历了base64压缩 - byte[] bytes = ZstdUtils.compress(contentBytes); - byte[] base64Bytes = Base64.getEncoder().encode(bytes); - Map responseMap = new HashMap<>(8); - responseMap.put("response", base64Bytes); + responseMap.put("response", CompressHandler.compressResp(contentBytes)); tracerObject.add(responseMap); //此处可以对content做处理,然后再把content写回到输出流中 diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java index d05336f..0140478 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java @@ -83,9 +83,9 @@ public class CollectMode { public static final long COMPRESS_REQ_REQP = C_REQ | C_RESP; /** - * 压缩req+resp + * 压缩log+resp */ - public static final long COMPRESS_REQ_C_RESP = C_REQ | C_RESP; + public static final long COMPRESS_LOG_RESP = C_REQ | C_RESP; /** * 压缩req+log+resp diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java index 02b0bfe..d2a7b43 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -1,5 +1,11 @@ package com.jd.platform.jlog.common.handler; +import com.jd.platform.jlog.common.utils.ZstdUtils; + +import java.util.Base64; + +import static com.jd.platform.jlog.common.handler.CollectMode.*; + /** * @author tangbohu * @version 1.0.0 @@ -8,4 +14,55 @@ package com.jd.platform.jlog.common.handler; * @createTime 2022年03月10日 20:52:00 */ public class CompressHandler { + + /** + * 压缩策略 + */ + private long compress = COMPRESS_LOG_RESP; + + private int limit = 10000; + + + private CompressHandler(long compress, int limit){ + this.compress = compress; + this.limit = limit; + } + + private static volatile CompressHandler instance = null; + + public static void buildCompressHandler(long compress, int limit) { + instance = new CompressHandler(compress,limit); + } + + public static byte[] compressReq(byte[] contentBytes){ + if(instance == null || !isMatched(instance.compress, E_REQ)){ return contentBytes; } + return doCompress(contentBytes); + } + + public static byte[] compressLog(byte[] contentBytes){ + if(instance == null || !isMatched(instance.compress, E_LOG)){ return contentBytes; } + return doCompress(contentBytes); + } + + public static byte[] compressResp(byte[] contentBytes){ + if(instance == null || !isMatched(instance.compress, E_RESP)){ return contentBytes; } + return doCompress(contentBytes); + } + + private static byte[] doCompress(byte[] contentBytes){ + //最终的要发往worker的response，经历了base64压缩 + byte[] bytes = ZstdUtils.compress(contentBytes); + return Base64.getEncoder().encode(bytes); + } + + + public long getCompress() { + return compress; + } + + public void setCompress(long compress) { + this.compress = compress; + } + + } diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java index 1e1507c..1240f34 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java @@ -49,10 +49,6 @@ public class TagConfig implements Serializable { */ private long extract = EXTRACT_ALL; - /** - * 压缩策略 - */ - private long compress = COMPRESS_ALL; public List getReqTags() { @@ -111,14 +107,6 @@ public class TagConfig implements Serializable { this.extract = extract; } - public long getCompress() { - return compress; - } - - public void setCompress(long compress) { - this.compress = compress; - } - @Override public String toString() { @@ -130,7 +118,6 @@ public class TagConfig implements Serializable { ", delimiter='" + delimiter + '\'' + ", join='" + join + '\'' + ", extract='" + extract + '\'' + - ", compress='" + compress + '\'' + '}'; } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java index 404fdce..828d726 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java @@ -46,15 +46,15 @@ public class TagHandler { private long extract; - private static volatile TagHandler INSTANCE = null; + private static volatile TagHandler instance = null; /** * 构建标签处理器 * @param tagConfig 配置类 */ - public static void buildHandler(TagConfig tagConfig) { + public static void buildTagHandler(TagConfig tagConfig) { - if(tagConfig.getExtract() == SUSPEND && tagConfig.getCompress() == SUSPEND ){ + if(tagConfig.getExtract() == SUSPEND){ return; } @@ -75,8 +75,8 @@ public class TagHandler { handler.delimiter = tagConfig.getDelimiter(); handler.delimiterLen = tagConfig.getDelimiter().length(); handler.join = tagConfig.getJoin(); - INSTANCE = handler; - LOGGER.info("构建标签处理器单例完成:{}",INSTANCE.toString()); + instance = handler; + LOGGER.info("构建标签处理器单例完成:{}",instance.toString()); } @@ -88,12 +88,12 @@ public class TagHandler { */ public static Map extractReqTag(Map params, @NotNull Map ext) { - if(INSTANCE == null || !isMatched(INSTANCE.extract, E_REQ)){ return null; } + if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } - System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(INSTANCE.reqTags)); + System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); - Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); - for (String tag : INSTANCE.reqTags) { + Map requestMap = new HashMap<>(instance.reqTags.size()); + for (String tag : instance.reqTags) { Object val = ext.get(tag); if(val != null){ requestMap.put(tag, val); @@ -115,17 +115,17 @@ public class TagHandler { * @return tags */ public static Map extractLogTag(String content) { - if(INSTANCE == null || !isMatched(INSTANCE.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ + if(instance == null || !isMatched(instance.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ return null; } Map tagMap = new HashMap<>(3); - Matcher m = INSTANCE.pattern.matcher(content); + Matcher m = instance.pattern.matcher(content); while (m.find()) { - String str = m.group().substring(INSTANCE.delimiterLen, m.group().length() - INSTANCE.delimiterLen); - if(str.contains(INSTANCE.join)){ - String[] arr = str.split(INSTANCE.join); - if(INSTANCE.logTags.contains(arr[0])){ + String str = m.group().substring(instance.delimiterLen, m.group().length() - instance.delimiterLen); + if(str.contains(instance.join)){ + String[] arr = str.split(instance.join); + if(instance.logTags.contains(arr[0])){ tagMap.put(arr[0], arr[1]); } }else if(str.length() < TAG_NORMAL_KEY_MAX_LEN){ @@ -148,12 +148,12 @@ public class TagHandler { */ public static Map extractRespTag(Map resp) { - if(INSTANCE == null || !isMatched(INSTANCE.extract, E_REQ)){ return null; } + if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } - System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(INSTANCE.reqTags)); + System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); - Map requestMap = new HashMap<>(INSTANCE.reqTags.size()); - for (String tag : INSTANCE.reqTags) { + Map requestMap = new HashMap<>(instance.reqTags.size()); + for (String tag : instance.reqTags) { Object val = resp.get(tag); if(val != null){ requestMap.put(tag, val); @@ -171,8 +171,8 @@ public class TagHandler { * @param tagConfig 新的配置 */ public synchronized static void refresh(TagConfig tagConfig) { - INSTANCE = null; - buildHandler(tagConfig); + instance = null; + buildTagHandler(tagConfig); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index 26450e5..b7371a3 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -230,7 +230,7 @@ public class FileConfigurator implements Configurator { PROPERTIES.clear(); FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props)); LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); - TagHandlerBuilder.refresh(); + // TagHandlerBuilder.refresh(); } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java index 30fef46..e93a13f 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -2,9 +2,11 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.handler.CollectMode; +import com.jd.platform.jlog.common.utils.ZstdUtils; import java.io.FileInputStream; import java.io.IOException; +import java.util.Base64; /** * @author tangbohu @@ -19,11 +21,28 @@ public final class FileNode { private long lastModity; public static void main(String[] args) throws IOException { - - System.out.println(CollectMode.COMPRESS_ALL); + String str = "这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是"+"{\"@level\":\"info\",\"@message\":\"response json: \",\"@module\":\"testing-platform.cfeature_plugin\",\"@timestamp\":\"2022-03-10T19:37:55.181928+08:00\",这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是" + + "\"EXTRA_VALUE_AT_END\":{\"data\":{\"accountAddr\":\"bg\",\"accountArea\":\"\",\"accountBank\":\"012\",\"accountBankBranch\":\"\",\"accountBankBranchNo\":\"\",\"accountBankName\":\"BBVA BANCOMER SA\",\"accountBranch\":\"\",\"accountCardDigit\":\"\",\"accountCardType\":null,\"accountCardno\":\"012420015684144165\",\"accountCity\":\"\",\"accountCompany\":\"\",\"accountFirstName\":\"\",\"accountGrantPhoto\":{\"keys\":[\"do1_tQxmThhy446Wf8JLHK5E\"],\"urls\":[\"http://img1.didiglobal.com/static/sailing_private_b/do1_tQxmThhy446Wf8JLHK5E?expire=1647517074\\u0026signiture=iTo-tiI21Ysdn7n7Y6y2YGs36avRi0cfLUpRl0rlE-g=\"]},\"accountHolder\":\"test_accountHolder\",\"accountHolderId\":\"\",\"accountLastName\":null,\"accountType\":0,\"addr\":\"test_addr\",\"addrAlph\":\"\",\"applyId\":\"5764607808352618589\",\"applyStatus\":0,\"applyType\":1,\"areaName\":\"Area0\",\"arrBusinessAttr\":[\"140\"],\"arrTags\":[],\"authorizationInfo\":{\"payment\":{\"authorizedPics\":null," + + "\"extraFiles\":null,\"pics\":null},\"sign\":{\"authorizedPics\":null,\"extraFiles\":null,\"pics\":null}},\"autoSwitch\":\"0\",\"avgPrice\":{\"max\":0,\"min\":0},\"avgProduceTime\":600,\"bLicenseAddr\":\"\",\"bLicenseBrand\":\"\",\"bLicenseExpireFlag\":61,\"bLicenseExpireTime\":\"1970-01-01 08:00:00\",\"bLicenseId\":\"\",\"bLicenseLpName\":\"\",\"bLicensePic\":\"\",\"bLicensePicUrl\":\"\",\"bLicenseStartTime\":\"1970-01-01 08:00:00\",\"bLicenseStatus\":\"3\",\"bLicenseValidType\":0,\"bankAgentDigit\":\"\",\"bankAgentno\":\"\",\"bdName\":\"sim_data_01(simdata01_test_v)\",\"bdPhone\":\"\",\"bizDayTime\":null,\"bizDayTimeNonAffiliate\":\"\",\"bizStatus\":1,\"bizTime\":\"{\\\"day\\\":[1,1,1,1,1,1,1],\\\"time\\\":[{\\\"begin\\\":\\\"10:00\\\",\\\"end\\\":\\\"18:00\\\"}]}\",\"bizoppDocInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"" + + "licensePics\":null}},\"bizoppInfo\":{\"bussinessHours\":\"\",\"deliveryDistanceTime\":\"\",\"deliveryPrice\":0,\"itemCount\":null,\"minPrice\":0,\"priceRange\":null,\"ratingCount\":\"\",\"ratingValue\":\"\"},\"bizoppStatus\":5,\"blicenseShow\":1,\"brand\":\"\",\"brandId\":0,\"businessType\":1,\"button\":{\"addVisitButton\":1,\"applyOpenButton\":0,\"applySignButton\":0,\"approveButton\":0,\"batchAllotButton\":1,\"businessInformationViewButton\":1,\"cancelApplyButton\":0,\"conSignPadButton\":0,\"confirmButton\":0,\"controlOperationButton\":1,\"copyMsgGatherUrlButton\":0,\"deliveryPadButton\":1,\"deliverySettingButton\":1,\"editButton\":0,\"financialOperationButton\":1,\"giveUp\":0,\"msgGatherButton\":0,\"msgGatherEditButton\":0,\"msgGatherPlusButton\":0,\"orderInfoButton\":1,\"pickIn\":0,\"sendMsgGatherButton\":0,\"switchToNonAffiliateButton\":0,\"tagsButton\":1,\"topEatsButton\":0,\"updateAbnormalTagsButton\":1,\"updateAddressButton\":1,\"updateBusinessInfoButton\":1," + + "\"updateCategoryButton\":1,\"updateNormalTag\":1,\"updateShopBaseInfo\":1,\"updateShopInfo\":1,\"uploadPictureButton\":1,\"viewOpenButton\":1,\"visitListButton\":1},\"cLicenseAddr\":\"\",\"cLicenseBrand\":\"\",\"cLicenseExpireFlag\":61,\"cLicenseExpireTime\":\"1970-01-01 08:00:00\",\"cLicenseId\":\"\",\"cLicenseLpName\":\"\",\"cLicensePic\":\"\",\"cLicensePicUrl\":\"\",\"cLicenseStatus\":\"3\",\"cLicenseValidType\":0,\"category\":[],\"categoryIds\":[],\"channel\":1,\"checkStatus\":0,\"city\":52080200,\"cityId\":52080200,\"clicenseShow\":1,\"clusterId\":5764607713452294000,\"clusterName\":\"\",\"commissionJson\":[],\"competeBizTime\":[],\"competeItemMenuUrl\":[],\"completeStatus\":1,\"contactCallingCode\":\"\",\"contactName\":\"test_partyaContact\",\"contactPhone\":\"15512300000\",\"contractorId\":\"5764607748000712588\",\"contractorStatus\":3,\"cosTrans\":0,\"country\":\"MX\",\"countryCode\":\"\",\"createTime\":\"2022-02-16 19:39:42\",\"creatorUserName\":\"simdata01_test_v\"," + + "\"currency\":{\"display\":\"MX$0.00\",\"number\":\"0.00\",\"position\":0,\"sign\":\"\",\"symbol\":\"MX$\"},\"decorateId\":0,\"deliverDistance\":0,\"deliverType\":2,\"deviceFlagbin\":3,\"deviceFlagbinNeedPad\":1,\"displayable\":0,\"displayableNonAffiliate\":0,\"docFlagbin\":0,\"docId\":\"5764607687590152047\",\"docInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"licensePics\":null}},\"editTime\":\"2022-02-16 19:40:14\",\"environmentalPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"extendMaterial\":\"\",\"extraInfo\":{\"bankFlow\":{\"files\":null,\"pics\":null},\"other\":{\"files\":null,\"pics\":null}},\"filterTags\":[{\"text\":\"正常门店\",\"type\":1}],\"filterTagsValue\":0,\"flagbin\":0,\"foodSafe\":\"\",\"foodSafePic\":\"\",\"foodSafePicUrl\":\"\",\"foodSafeShow\":1," + + "\"frontPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"],\"grade\":\"\",\"headImg\":\"\",\"isDeliverCash\":1,\"isEnableCash\":1,\"isEnablePaypay\":1,\"isEnablePos\":0,\"isFoodCity\":0,\"isIllegal\":0,\"isNeedPad\":0,\"isRate\":2,\"isScopeDelivery\":1,\"itemFiles\":null,\"itemPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_iad8ykReSCUcGh7sUJye\"],\"ka\":1,\"kpContactTime\":null,\"kpDepartment\":\"\",\"kpEmail\":\"test_data_platform@didiglobal.com\",\"kpLastName\":\"\",\"kpLastNameAlph\":\"\",\"kpName\":\"test_kpName\",\"kpNameAlph\":\"\",\"kpOutId\":\"\",\"kpPhone\":\"15512300001\",\"kpPhoneExtra\":null,\"kpPics\":{\"keys\":[\"https://img0.didiglobal.com/static/soda_public/do1_BXtt9g8f8dfiecSHC1dZ\"],\"urls\":[false]},\"kpSource\":\"\",\"lat\":\"28.7370616\",\"lng\":\"-106.1306375\",\"localTaxId\":\"\",\"localTaxPayer\":\"\",\"localTaxPayerAddr\":\"\",\"localTaxPayerIdType\":0,\"localTaxPaymentMethod\":\"\"," + + "\"localTaxPics\":\"\",\"localTaxVatPayment\":0,\"logo\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"logoImg\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"lpLicenseId\":\"\",\"lpLicensePicBack\":\"\",\"lpLicensePicBackUrl\":\"\",\"lpLicensePicFront\":\"\",\"lpLicensePicFrontUrl\":\"\",\"lpLicensePics\":\"\",\"lpLicensePicsUrl\":\"\",\"lpLicenseStatus\":\"3\",\"lpLicenseType\":\"0\",\"lpName\":\"\",\"mainCategory\":\"\",\"manualCategory\":[],\"minPrice\":0,\"monthSales\":0,\"monthSold\":0,\"msgGatherUrl\":\"\",\"name\":\"test_1645011556_kFvjwPijhdxkmmgRXKC\",\"nameAlph\":\"\",\"notifyEmail\":\"\",\"notifyPhone\":\"\",\"objBusinessAttr\":[],\"objTags\":[],\"officialUrl\":\"\",\"open\":1,\"openCheck\":[{\"button\":\"已添加\",\"text\":\"联系电话\",\"type\":1},{\"button\":\"已填写\",\"text\":\"配送信息\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店头图\",\"type\":1},{\"button\":\"未完成\",\"text\":\"可售菜品数量\"," + + "\"type\":0},{\"button\":\"已通过\",\"text\":\"营业时间段\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店主营品类\",\"type\":1},{\"button\":\"已添加\",\"text\":\"出餐时长\",\"type\":1}],\"openCheckNonAffiliate\":null,\"orderConfirmMethod\":\"0\",\"padInfo\":{\"isNeedPad\":0,\"isPadOnlineSign\":1},\"padStatus\":0,\"payVerifyReason\":\"\",\"payVerifyStatus\":0,\"permAddr\":\"\",\"permNo\":\"\",\"phone\":\"164501155695\",\"poi\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiId\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiName\":\"Arcos de Terragona 16934, Villa del Nte, 31137 Chihuahua, Chih., México\",\"postCode\":\"\",\"produceTimeMax\":\"0\",\"rank\":5,\"repeatShopId\":\"\",\"reprePhone\":\"\",\"riderAreaName\":\"\",\"riderClusterId\":52080200,\"score\":0,\"secondCategory\":\"\",\"selectCityId\":52080200,\"settledStatus\":0,\"settledTags\":0,\"shopId\":\"5764607747732277132\",\"shopLayerLevel\":\"\",\"shopLevel\":\"\",\"shopOnlineStatus\":3,\"shopOnlineStatusDesc\":\"需手工开启营业\"," + + "\"shopOnlineStatusMsg\":\"歇业中\",\"shopPhone\":\"\",\"shopPics\":{\"environmentalPhoto\":[\"/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"frontPhoto\":[\"/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"]},\"shopStatus\":3,\"shopTag\":0,\"signPic\":null,\"signType\":1,\"signedTime\":1645011613,\"source\":2,\"startupCost\":500000,\"startupCostLimit\":50000,\"startupCostNum\":0,\"startupCostProportion\":1000,\"startupExecWeek\":1,\"tags\":0,\"taxId\":\"234234123112\",\"taxIdType\":1,\"taxPayer\":\"\",\"taxPayerAddr\":\"\",\"taxPayerIdType\":0,\"taxPaymentMethod\":\"\",\"taxPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_Gip4umImni45IUHPTvQQ\"],\"taxVatPayment\":0,\"thirdCategory\":\"\",\"timezone\":\"\",\"todayBizTime\":[],\"type\":1,\"updateTime\":\"2022-03-10 15:46:20\",\"visible\":false,\"visitStatus\":0},\"errmsg\":\"ok\",\"errno\":0,\"machine\":\"\",\"reqParams\":{\"locale\":\"zh-CN\",\"shopId\":\"5764607747732277132\"},\"time\":1646912275," + + "\"traceId\":\"s060310x1646912272\"},\"timestamp\":\"2022-03-10T19:37:55.180+0800\"}"; + System.out.println(str.length()); + System.out.println(str.getBytes().length); + System.out.println("======="); + byte[] bt = ZstdUtils.compress(str.getBytes()); + System.out.println(bt.length); + System.out.println(Base64.getEncoder().encode(bt).length); + + System.out.println("======="); + System.out.println(CollectMode.COMPRESS_LOG_RESP); System.out.println(CollectMode.EXTRACT_ALL); - System.out.println(); JcProperties properties = new JcProperties(); String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java similarity index 74% rename from config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java rename to config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java index c3bde54..cd7868a 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/TagHandlerBuilder.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java @@ -1,5 +1,6 @@ package com.jd.platform.jlog.core; +import com.jd.platform.jlog.common.handler.CompressHandler; import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.common.handler.TagHandler; @@ -10,15 +11,16 @@ import com.jd.platform.jlog.common.handler.TagHandler; * @Description TODO * @createTime 2022年03月05日 22:07:00 */ -public class TagHandlerBuilder { +public class HandlerBuilder { - public static void buildTagHandler(TagConfig tagConfig, Configurator configurator){ + public static void buildHandler(TagConfig tagConfig, Configurator configurator){ if(tagConfig == null){ tagConfig = buildTagConfigByConfigurator(configurator); } - TagHandler.buildHandler(tagConfig); + TagHandler.buildTagHandler(tagConfig); + CompressHandler.buildCompressHandler(configurator.getLong("compress")); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java index 9d2368e..a5b58eb 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java @@ -91,7 +91,7 @@ public class JcProperties extends Properties { field.setShort(model, Short.valueOf(properties.getString(fillName))); break; case "int": - field.setInt(model, properties.getLong(fillName).intValue()) ; + field.setInt(model, Integer.parseInt(properties.getString(fillName))) ; break; case "long": field.setLong(model, properties.getLong(fillName)); diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties index b924a69..28575e6 100644 --- a/example/src/main/resources/application.properties +++ b/example/src/main/resources/application.properties @@ -7,4 +7,7 @@ tag-config.regex="" tag-config.delimiter="|" tag-config.join="=" tag-config.extract=41 -tag-config.compress=81 + +compress=68 +limit=10000 + -- Gitee From 1755ec1b10f2d946ce07f748e30e914795ff1cfe Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Sun, 13 Mar 2022 14:14:47 +0800 Subject: [PATCH 18/22] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/common/constant/Constant.java | 14 +- .../jlog/common/handler/CompressHandler.java | 27 +++- .../jlog/common/handler/TagHandler.java | 39 +++-- .../platform/jlog/common/utils/ZstdUtils.java | 117 +++++++++++++++ .../jlog/core/ConfigChangeListener.java | 3 +- .../jd/platform/jlog/core/Configurator.java | 1 + .../platform/jlog/core/FileConfigurator.java | 23 +-- .../com/jd/platform/jlog/core/FileNode.java | 38 ++++- .../jd/platform/jlog/core/GzipCompress.java | 49 ++++++ .../jd/platform/jlog/core/HandlerBuilder.java | 4 +- .../jd/platform/jlog/core/JcProperties.java | 32 +++- .../com/jd/platform/jlog/core/NodeBak.java | 142 ------------------ .../com/jd/platform/jlog/core/ObjObj.java | 45 ------ .../com/jd/platform/jlog/core/Server.java | 32 ---- .../src/main/resources/application.properties | 20 ++- .../com/jd/platform/jlog/test/Common.java | 5 +- .../jlog/test/FileConfiguratorTest.java | 6 +- 17 files changed, 314 insertions(+), 283 deletions(-) create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java delete mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java delete mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java delete mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java index 1c069f4..71dbafc 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java @@ -42,7 +42,19 @@ public class Constant { /** * 符合正则但不构成kv结构的普通日志，如「xx查询为空」 */ - public static String TAG_NORMAL_KEY = "normal"; + public static final String TAG_NORMAL_KEY = "normal"; + + /** + * 小于1000不考虑压缩 + */ + public static final long MIN = 1000; + + + /** + * 默认大于10000才考虑压缩 + */ + public static final long THRESHOLD = 10000; + /** * 正则里需要特殊处理的字符串 diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java index d2a7b43..82b70f4 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -4,6 +4,8 @@ import com.jd.platform.jlog.common.utils.ZstdUtils; import java.util.Base64; +import static com.jd.platform.jlog.common.constant.Constant.MIN; +import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD; import static com.jd.platform.jlog.common.handler.CollectMode.*; /** @@ -18,20 +20,30 @@ public class CompressHandler { /** * 压缩策略 */ - private long compress = COMPRESS_LOG_RESP; + private long compress; - private int limit = 10000; + /** + * 超过limit的才压缩 + */ + private long threshold; - private CompressHandler(long compress, int limit){ + private CompressHandler(long compress, long threshold){ this.compress = compress; - this.limit = limit; + this.threshold = threshold; } private static volatile CompressHandler instance = null; - public static void buildCompressHandler(long compress, int limit) { - instance = new CompressHandler(compress,limit); + + public static void buildCompressHandler(Long compress, Long threshold) { + if(compress == null || compress < 1){ + compress = COMPRESS_LOG_RESP; + } + if( threshold == null || threshold < MIN){ + threshold = THRESHOLD; + } + instance = new CompressHandler(compress, threshold); } public static byte[] compressReq(byte[] contentBytes){ @@ -50,6 +62,9 @@ public class CompressHandler { } private static byte[] doCompress(byte[] contentBytes){ + if(contentBytes.length < instance.threshold){ + return contentBytes; + } //最终的要发往worker的response，经历了base64压缩 byte[] bytes = ZstdUtils.compress(contentBytes); return Base64.getEncoder().encode(bytes); diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java index 828d726..0f4619a 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java @@ -30,11 +30,11 @@ public class TagHandler { private static final Logger LOGGER = LoggerFactory.getLogger(TagHandler.class); - private Set reqTags; + private List reqTags; - private Set logTags; + private List logTags; - private Set respTags; + private List respTags; private String delimiter = "|"; @@ -60,12 +60,12 @@ public class TagHandler { TagHandler handler = new TagHandler(); handler.extract = tagConfig.getExtract(); - handler.reqTags = new HashSet<>(tagConfig.getReqTags()); - handler.logTags = new HashSet<>(tagConfig.getLogTags()); - handler.respTags = new HashSet<>(tagConfig.getRespTags()); + handler.reqTags = tagConfig.getReqTags(); + handler.logTags = tagConfig.getLogTags(); + handler.respTags = tagConfig.getRespTags(); String regex = tagConfig.getRegex(); - if(StringUtil.isNotEmpty(regex)){ + if(StringUtil.isNotBlank(regex)){ handler.pattern = Pattern.compile(regex); }else{ String escapeDelimiter = ConfigUtil.escapeExprSpecialWord(tagConfig.getDelimiter()); @@ -76,6 +76,7 @@ public class TagHandler { handler.delimiterLen = tagConfig.getDelimiter().length(); handler.join = tagConfig.getJoin(); instance = handler; + System.out.println("### instance == > "+instance.toString()); LOGGER.info("构建标签处理器单例完成:{}",instance.toString()); } @@ -91,6 +92,8 @@ public class TagHandler { if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); + System.out.println("### .params:"+JSON.toJSONString(params)); + System.out.println("### .ext:"+JSON.toJSONString(ext)); Map requestMap = new HashMap<>(instance.reqTags.size()); for (String tag : instance.reqTags) { @@ -118,6 +121,8 @@ public class TagHandler { if(instance == null || !isMatched(instance.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ return null; } + System.out.println("### INSTANCE:"+instance.toString()); + System.out.println("### .content:"+content); Map tagMap = new HashMap<>(3); Matcher m = instance.pattern.matcher(content); @@ -192,8 +197,15 @@ public class TagHandler { // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); public static List extractTest(String content) { + TagConfig cfg = new TagConfig(); + cfg.setDelimiter("|"); + cfg.setJoin("="); + + buildTagHandler(cfg); + List list = new ArrayList<>(); - Matcher m = BRACKET_PATTERN.matcher(content); + // Matcher m = BRACKET_PATTERN.matcher(content); + Matcher m = instance.pattern.matcher(content); while (m.find()) { list.add(m.group().substring(1, m.group().length() - 1)); // list.add(m.group().substring(2, m.group().length() - 2)); @@ -207,16 +219,21 @@ public class TagHandler { // static Pattern BRACKET_PATTERN = Pattern.compile("(\\|\\|[\\w\\W]*?\\|\\|)"); static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)"); - static String str1 = "||a=1||b=2||qwewe"; + static String str1 = "|errno=val3||node=val4||这是随便的log|"; static String str2 = "||a=1||b=2||qwewe||"; static String str3 = "||a=1||eee||b=2"; static String str4 = "||a=1||eee||b=2||"; public static void main(String[] args) { + + + //### INSTANCE:TagHandler{reqTags=[uid, url], logTags=[node, bizType], + // delimiter='|', delimiterLen=1, join='"="', pattern="", extract=41} + //### .content:|errno=val3||node=val4||这是随便的log| System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1))); - System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2))); + /* System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2))); System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3))); - System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4))); + System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4)));*/ } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java index 3a125d0..d4757eb 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java @@ -2,6 +2,11 @@ package com.jd.platform.jlog.common.utils; import com.github.luben.zstd.Zstd; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + /** * zstd压缩工具类 * @@ -39,4 +44,116 @@ public class ZstdUtils { return ob; } + + + /** + * 解压 + */ + public static byte[] decompressBytes2(byte[] bytes) { + int size = (int) Zstd.decompressedSize(bytes); + System.out.println("size-> "+size); + byte[] ob = new byte[size]; + Zstd.decompress(ob, bytes); + + return ob; + } + + public static void main(String[] args) throws InterruptedException, ClassNotFoundException, IllegalAccessException, InstantiationException { + + String s = "-75"; + char firstChar = s.charAt(0); + System.out.println(firstChar); + if (firstChar < '0') { // Possible leading "+" or "-" + if (firstChar == '-') { + System.out.println("========="); + } else if (firstChar != '+') + System.out.println("eeee"); + + } + // System.out.println(Integer.parseInt("-75")); + String str1 = "1a"; + String str2 = "0a"; + + String dbStr = Arrays.toString(compress(str1.getBytes())); + System.out.println(dbStr); + String[] arr0 = dbStr.split(","); + + byte[] bt = new byte[arr0.length]; + for (int i = 0; i < arr0.length; i++) { + if(i == 0){ + String first = arr0[0].replace("[", ""); + bt[i] = Byte.valueOf(first); + continue; + } + if(i == arr0.length-1){ + String last = arr0[arr0.length-1].replace("]", ""); + bt[i] = Byte.valueOf(last.trim()); + continue; + } + System.out.println(arr0[i]+" len => "+arr0[i].length()); + bt[i] = Byte.parseByte(arr0[i].trim()); + } + + + System.out.println(Arrays.toString(bt)); + // System.out.println(Arrays.toString(compress(str2.getBytes()))); + + AtomicInteger fail = new AtomicInteger(0); + AtomicInteger ok = new AtomicInteger(0); + + for (int j = 0; j < 1; j++) { + new Thread(() -> { + for (int i = 0; i < 100; i++) { + int id = new Random().nextInt(99000000); + /* String ml = String.valueOf(getRandomChar()) +id+getRandomChar(); + String ml2 = LocalDateTime.now().toString()+ getRandomChar(); + ml = ml2 + ml;*/ + // 压缩后的 + String ml = ""+i; + byte[] mlc1= compress(ml.getBytes()); + + //存进去new String(mlc1) + byte[] resul = new String(mlc1).getBytes(); + System.out.println(Arrays.toString(new String(mlc1).getBytes())); + byte[] arr = new byte[9]; + arr[0] = 40; + arr[1] = -17; + arr[2] = -65; + arr[3] = -67; + arr[4] = 47; + arr[5] = -17; + arr[6] = -65; + arr[7] = -67; + arr[8] = 32; + // System.out.println(new String(arr)); + + if(resul[0]==40 + && resul[1]==-17 + && resul[2]==-65 + && resul[3]==-67 + && resul[4]==47 + && resul[5]==-17 + && resul[6]==-65 + && resul[7]==-67 + && resul[8]==32 ){ + ok.incrementAndGet(); + }else{ + fail.incrementAndGet(); + } + } + }).start(); + } + + + Thread.sleep(30000); + + + System.out.println("ok == > "+ok.get()); + System.out.println("fail == > "+fail.get()); + + } + + public static char getRandomChar() { + return (char) (0x4e00 + (int) (Math.random() * (0x9fa5 - 0x4e00 + 1))); + } } \ No newline at end of file diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java index 5c790ee..9d337a2 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java @@ -16,6 +16,7 @@ package com.jd.platform.jlog.core; +import io.netty.util.concurrent.DefaultThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +34,7 @@ public interface ConfigChangeListener { ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(1, 1, - Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), Executors.defaultThreadFactory()); + Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("configListener", 1)); /** diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index 4ae742d..fd25c7b 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -94,6 +94,7 @@ public interface Configurator { * 移除监听器 * @param node 节点 file or dir */ + @Deprecated void removeConfigListener(String node); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index b7371a3..8d97407 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -15,6 +15,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import static com.jd.platform.jlog.core.ConfigChangeListener.EXECUTOR_SERVICE; import static com.jd.platform.jlog.core.Constant.*; @@ -136,9 +137,9 @@ public class FileConfigurator implements Configurator { LOGGER.info("没有要监听的key了 关闭线程池"); FILELISTENER.onShutDown(); try { - if(!FILELISTENER.executor.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ + if(!EXECUTOR_SERVICE.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ // 超时的时候向线程池中所有的线程发出中断(interrupted)。 - FILELISTENER.executor.shutdownNow(); + EXECUTOR_SERVICE.shutdownNow(); } } catch (InterruptedException e) { e.printStackTrace(); @@ -151,13 +152,7 @@ public class FileConfigurator implements Configurator { @Override public List getConfigByPrefix(String prefix) { - - Object val = PROPERTIES.get(prefix); - - if(val != null){ - return FastJsonUtils.toList(String.valueOf(val), String.class); - } - return null; + return PROPERTIES.getStrList(prefix); } @@ -171,10 +166,6 @@ public class FileConfigurator implements Configurator { class FileListener implements ConfigChangeListener { - private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, - TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), - new DefaultThreadFactory("fileListener", 1)); - FileListener() { } @@ -192,12 +183,6 @@ public class FileConfigurator implements Configurator { } } } - - - @Override - public ExecutorService getExecutorService() { - return executor; - } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java index e93a13f..9756593 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -2,10 +2,12 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.handler.CollectMode; +import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.common.utils.ZstdUtils; import java.io.FileInputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Base64; /** @@ -21,6 +23,20 @@ public final class FileNode { private long lastModity; public static void main(String[] args) throws IOException { + + String a = "汉字"; + System.out.println("a byte length："+a.getBytes().length); + + byte[] bt1 = ZstdUtils.compress(a.getBytes()); + String ckStr = new String(bt1); + String newStr1 = ZstdUtils.decompress(bt1); + System.out.println("老 字符串 bt："+ Arrays.toString(a.getBytes())); + System.out.println("新 字符串 bt："+ Arrays.toString(ckStr.getBytes())); + + + System.out.println("新字符串："+newStr1); + System.out.println("新字符串的byte长度："+newStr1.getBytes().length); + String str = "这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是"+"{\"@level\":\"info\",\"@message\":\"response json: \",\"@module\":\"testing-platform.cfeature_plugin\",\"@timestamp\":\"2022-03-10T19:37:55.181928+08:00\",这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是" + "\"EXTRA_VALUE_AT_END\":{\"data\":{\"accountAddr\":\"bg\",\"accountArea\":\"\",\"accountBank\":\"012\",\"accountBankBranch\":\"\",\"accountBankBranchNo\":\"\",\"accountBankName\":\"BBVA BANCOMER SA\",\"accountBranch\":\"\",\"accountCardDigit\":\"\",\"accountCardType\":null,\"accountCardno\":\"012420015684144165\",\"accountCity\":\"\",\"accountCompany\":\"\",\"accountFirstName\":\"\",\"accountGrantPhoto\":{\"keys\":[\"do1_tQxmThhy446Wf8JLHK5E\"],\"urls\":[\"http://img1.didiglobal.com/static/sailing_private_b/do1_tQxmThhy446Wf8JLHK5E?expire=1647517074\\u0026signiture=iTo-tiI21Ysdn7n7Y6y2YGs36avRi0cfLUpRl0rlE-g=\"]},\"accountHolder\":\"test_accountHolder\",\"accountHolderId\":\"\",\"accountLastName\":null,\"accountType\":0,\"addr\":\"test_addr\",\"addrAlph\":\"\",\"applyId\":\"5764607808352618589\",\"applyStatus\":0,\"applyType\":1,\"areaName\":\"Area0\",\"arrBusinessAttr\":[\"140\"],\"arrTags\":[],\"authorizationInfo\":{\"payment\":{\"authorizedPics\":null," + "\"extraFiles\":null,\"pics\":null},\"sign\":{\"authorizedPics\":null,\"extraFiles\":null,\"pics\":null}},\"autoSwitch\":\"0\",\"avgPrice\":{\"max\":0,\"min\":0},\"avgProduceTime\":600,\"bLicenseAddr\":\"\",\"bLicenseBrand\":\"\",\"bLicenseExpireFlag\":61,\"bLicenseExpireTime\":\"1970-01-01 08:00:00\",\"bLicenseId\":\"\",\"bLicenseLpName\":\"\",\"bLicensePic\":\"\",\"bLicensePicUrl\":\"\",\"bLicenseStartTime\":\"1970-01-01 08:00:00\",\"bLicenseStatus\":\"3\",\"bLicenseValidType\":0,\"bankAgentDigit\":\"\",\"bankAgentno\":\"\",\"bdName\":\"sim_data_01(simdata01_test_v)\",\"bdPhone\":\"\",\"bizDayTime\":null,\"bizDayTimeNonAffiliate\":\"\",\"bizStatus\":1,\"bizTime\":\"{\\\"day\\\":[1,1,1,1,1,1,1],\\\"time\\\":[{\\\"begin\\\":\\\"10:00\\\",\\\"end\\\":\\\"18:00\\\"}]}\",\"bizoppDocInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"" + @@ -32,12 +48,21 @@ public final class FileNode { "\"type\":0},{\"button\":\"已通过\",\"text\":\"营业时间段\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店主营品类\",\"type\":1},{\"button\":\"已添加\",\"text\":\"出餐时长\",\"type\":1}],\"openCheckNonAffiliate\":null,\"orderConfirmMethod\":\"0\",\"padInfo\":{\"isNeedPad\":0,\"isPadOnlineSign\":1},\"padStatus\":0,\"payVerifyReason\":\"\",\"payVerifyStatus\":0,\"permAddr\":\"\",\"permNo\":\"\",\"phone\":\"164501155695\",\"poi\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiId\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiName\":\"Arcos de Terragona 16934, Villa del Nte, 31137 Chihuahua, Chih., México\",\"postCode\":\"\",\"produceTimeMax\":\"0\",\"rank\":5,\"repeatShopId\":\"\",\"reprePhone\":\"\",\"riderAreaName\":\"\",\"riderClusterId\":52080200,\"score\":0,\"secondCategory\":\"\",\"selectCityId\":52080200,\"settledStatus\":0,\"settledTags\":0,\"shopId\":\"5764607747732277132\",\"shopLayerLevel\":\"\",\"shopLevel\":\"\",\"shopOnlineStatus\":3,\"shopOnlineStatusDesc\":\"需手工开启营业\"," + "\"shopOnlineStatusMsg\":\"歇业中\",\"shopPhone\":\"\",\"shopPics\":{\"environmentalPhoto\":[\"/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"frontPhoto\":[\"/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"]},\"shopStatus\":3,\"shopTag\":0,\"signPic\":null,\"signType\":1,\"signedTime\":1645011613,\"source\":2,\"startupCost\":500000,\"startupCostLimit\":50000,\"startupCostNum\":0,\"startupCostProportion\":1000,\"startupExecWeek\":1,\"tags\":0,\"taxId\":\"234234123112\",\"taxIdType\":1,\"taxPayer\":\"\",\"taxPayerAddr\":\"\",\"taxPayerIdType\":0,\"taxPaymentMethod\":\"\",\"taxPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_Gip4umImni45IUHPTvQQ\"],\"taxVatPayment\":0,\"thirdCategory\":\"\",\"timezone\":\"\",\"todayBizTime\":[],\"type\":1,\"updateTime\":\"2022-03-10 15:46:20\",\"visible\":false,\"visitStatus\":0},\"errmsg\":\"ok\",\"errno\":0,\"machine\":\"\",\"reqParams\":{\"locale\":\"zh-CN\",\"shopId\":\"5764607747732277132\"},\"time\":1646912275," + "\"traceId\":\"s060310x1646912272\"},\"timestamp\":\"2022-03-10T19:37:55.180+0800\"}"; - System.out.println(str.length()); - System.out.println(str.getBytes().length); - System.out.println("======="); + System.out.println("字符串长度： "+str.length()); + System.out.println("字符串byte长度："+str.getBytes().length); byte[] bt = ZstdUtils.compress(str.getBytes()); - System.out.println(bt.length); - System.out.println(Base64.getEncoder().encode(bt).length); + String newStr = ZstdUtils.decompress(bt); + System.out.println("新字符串："+newStr); + System.out.println("新字符串的byte长度："+newStr.getBytes().length); + + System.out.println("Zstd压缩后的byte长度："+bt.length); + + byte[] bs = GzipCompress.compress(str.getBytes()); + System.out.println("Gzip压缩后的byte长度："+bs.length); + System.out.println("Zstd压缩后的byte长度 + Gzip压缩后的byte长度："+GzipCompress.compress(bt).length); + + + System.out.println("Base64 后的byte长度: "+Base64.getEncoder().encode(bt).length); System.out.println("======="); System.out.println(CollectMode.COMPRESS_LOG_RESP); @@ -48,7 +73,8 @@ public final class FileNode { String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; properties.load(new FileInputStream(path)); System.out.println(JSON.toJSONString(properties)); - Server bean = properties.getBean("server", Server.class); + TagConfig bean = properties.getBean("tag-config", TagConfig.class); System.out.println(JSON.toJSONString(bean)); } + } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java new file mode 100644 index 0000000..22d75b6 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java @@ -0,0 +1,49 @@ +package com.jd.platform.jlog.core; + + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * The Data Compression Based on gzip. + * + */ +public class GzipCompress { + + public static byte[] compress(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip; + + try { + gzip = new GZIPOutputStream(out); + gzip.write(data); + gzip.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + return out.toByteArray(); + } + + public static byte[] uncompress(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(data); + + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[2048]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return out.toByteArray(); + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java index cd7868a..5d13796 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java @@ -20,7 +20,7 @@ public class HandlerBuilder { tagConfig = buildTagConfigByConfigurator(configurator); } TagHandler.buildTagHandler(tagConfig); - CompressHandler.buildCompressHandler(configurator.getLong("compress")); + CompressHandler.buildCompressHandler(configurator.getLong("compress"), configurator.getLong("threshold")); } @@ -36,6 +36,6 @@ public class HandlerBuilder { private static TagConfig buildTagConfigByConfigurator(Configurator configurator){ - return configurator.getObject("handler-config", TagConfig.class); + return configurator.getObject("tag-config", TagConfig.class); } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java index a5b58eb..b907677 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java @@ -21,8 +21,7 @@ import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst; */ public class JcProperties extends Properties { - public JcProperties() { - } + public JcProperties() {} public String getString(String key) { Object val = get(key); @@ -34,10 +33,10 @@ public class JcProperties extends Properties { public Long getLong(String key) { String val = getString(key); - if(StringUtil.isEmpty(val)){ + if(val == null){ return null; } - return Long.valueOf(val); + return Long.parseLong(val); } public List getStrList(String key) { @@ -65,6 +64,17 @@ public class JcProperties extends Properties { } + /** + * 只支持简单的对象形配置 + * @param model bean + * @param properties + * @param prefix + * @throws IllegalAccessException + * @throws ClassNotFoundException + * @throws InstantiationException + * @throws ParseException + */ + @Deprecated private void invoke(Object model, JcProperties properties, String prefix) throws IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException { @@ -112,8 +122,18 @@ public class JcProperties extends Properties { default: String tn = field.getType().getTypeName(); if("java.util.List".equals(tn)){ - String val = properties.getString(fillName); - field.set(model,FastJsonUtils.toList(val, String.class)); + String[] arr = fillName.split("\\["); + int index = 0; + String suffix; + String fastSuffix; + List list = new ArrayList<>(); + do{ + suffix = "["+index+"]"; + fastSuffix = "["+(index+1)+"]"; + list.add(properties.getString(arr[0]+suffix)); + index ++; + }while (properties.getString(arr[0]+fastSuffix) != null); + field.set(model, list); }else if("java.util.Map".equals(tn)){ String val = properties.getString(fillName); field.set(model,FastJsonUtils.toMap(val)); diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java deleted file mode 100644 index 3a5d521..0000000 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/NodeBak.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.jd.platform.jlog.core; - -import com.alibaba.fastjson.JSON; -import com.jd.platform.jlog.common.utils.StringUtil; -import org.yaml.snakeyaml.Yaml; - -import java.io.*; -import java.util.*; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName Node.java 包装类 用于以后的支持key->node->[listener,properties] - * @createTime 2022年03月02日 20:37:00 - */ -public class NodeBak implements Serializable { - - - private String name; - - private ConfigChangeListener listener; - - private Properties properties; - - private String type; - - private Object val; - - - static Properties newPro = new Properties(); - - - public static void main(String[] args) throws IOException { - - String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; - // newPro.load(); - read(path); - - System.out.println(JSON.toJSONString(newPro)); - } - - protected static Map toMap(Properties properties) { - Map result = new HashMap<>(10); - Enumeration propertyNames = properties.propertyNames(); - while (propertyNames.hasMoreElements()) { - String name = (String) propertyNames.nextElement(); - String value = properties.getProperty(name); - result.put(name, value); - } - return result; - } - - - - - public static void read(String path) { - - FileReader fr = null; - LineNumberReader lnr = null; - String str; - - try { - fr = new FileReader(path); - lnr = new LineNumberReader(fr); - - Stack stack = new Stack<>(); - HashMap tinyMap = new LinkedHashMap<>(); - List tinyList = new ArrayList<>(); - String temKey = ""; - while ((str = lnr.readLine()) != null) { - if(StringUtil.isBlank(str) || !str.contains("=")){ - continue; - } - String[] lineArr = str.split("="); - String key = lineArr[0]; - String val = lineArr[1]; - - if(!key.contains(".")){ - newPro.put(key,val.trim()); - continue; - } - - String[] keyArr = key.split("\\."); - - String lastTinyKey = keyArr[keyArr.length - 1]; - if(lastTinyKey.contains("[") & lastTinyKey.contains("]")){ - String[] lastArr = lastTinyKey.split("\\["); - for (int i = 0; i < keyArr.length - 1; i++) { - stack.push(keyArr[i]); - } - stack.push(lastArr[0]); - - /* if(!temKey.equals(lastArr[0])){ - temKey = lastArr[0]; - String out = stack.pop(); - stack.push(temKey); - }*/ - //list - tinyList.add(val); - newPro.put(keyArr[0],tinyList); - }else{ - for (int i = 0; i < keyArr.length - 1; i++) { - if(stack.empty() || !stack.peek().equals(keyArr[i])){ - stack.push(keyArr[i]); - } - } - - /* if(!temKey.equals(lastTinyKey)){ - temKey = lastTinyKey; - String out = stack.pop(); - System.out.println("out==> "+ out +" temKey==> '"+temKey ); - stack.push(temKey); - }*/ - // map - tinyMap.put(keyArr[1],val.trim()); - newPro.put(keyArr[0],tinyMap); - } - System.out.println(" -- stack==> " + JSON.toJSONString(stack) ); - - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (fr != null) { - try { - fr.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - if (lnr != null) { - try { - lnr.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - } - } - -} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java deleted file mode 100644 index 346f6b9..0000000 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ObjObj.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.jd.platform.jlog.core; - -import java.util.List; -import java.util.Map; - -public class ObjObj{ - private int last; - private String max; - - private List ids; - - private Map ms; - - public int getLast() { - return last; - } - - public void setLast(int last) { - this.last = last; - } - - public String getMax() { - return max; - } - - public void setMax(String max) { - this.max = max; - } - - public List getIds() { - return ids; - } - - public void setIds(List ids) { - this.ids = ids; - } - - public Map getMs() { - return ms; - } - - public void setMs(Map ms) { - this.ms = ms; - } -} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java deleted file mode 100644 index a958816..0000000 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Server.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jd.platform.jlog.core; - -public class Server{ - private int port; - private String addr; - private ObjObj objObj; - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public String getAddr() { - return addr; - } - - public void setAddr(String addr) { - this.addr = addr; - } - - - public ObjObj getObjObj() { - return objObj; - } - - public void setObjObj(ObjObj objObj) { - this.objObj = objObj; - } -} \ No newline at end of file diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties index 28575e6..3f355de 100644 --- a/example/src/main/resources/application.properties +++ b/example/src/main/resources/application.properties @@ -1,13 +1,19 @@ server.port=8085 -tag-config.reqTags=["uid","url"] -tag-config.logTags=["node","bizType"] -tag-config.respTags=["errno","msg"] -tag-config.regex="" -tag-config.delimiter="|" -tag-config.join="=" +tag-config.reqTags[0]=uid +tag-config.reqTags[1]=url + +tag-config.logTags[0]=node +tag-config.logTags[1]=bizType + +tag-config.respTags[0]=errno +tag-config.respTags[1]=msg +tag-config.delimiter=| +tag-config.join== tag-config.extract=41 compress=68 -limit=10000 +threshold=10000 +workers[0]=1 +workers[1]=2 diff --git a/example/src/test/java/com/jd/platform/jlog/test/Common.java b/example/src/test/java/com/jd/platform/jlog/test/Common.java index 3db0d8e..24d6e04 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/Common.java +++ b/example/src/test/java/com/jd/platform/jlog/test/Common.java @@ -1,6 +1,7 @@ package com.jd.platform.jlog.test; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.core.Configurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,8 +27,8 @@ public class Common { LOGGER.info("配置器类型：{}", configurator.getType()); String addr = configurator.getString("serverAddr"); LOGGER.info("配置器get addr：{}", addr); - String reqTags = configurator.getString("reqTags"); - LOGGER.info("配置器get reqTags：{}", reqTags); + TagConfig tagConfig = configurator.getObject("tagConfig", TagConfig.class); + LOGGER.info("配置器get tagConfig：{}", tagConfig.toString()); List workers = configurator.getConfigByPrefix("workers"); LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); } diff --git a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java index b8ec6ec..88e8267 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -51,7 +51,7 @@ public class FileConfiguratorTest { @Test public void testAddConfigListener() throws Exception { configurator.addConfigListener("/bakapplication.yml"); - String path = "/Users/didi/Desktop/jlog/example/target/classes/bakapplication.yml"; + String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; Properties props = new Properties(); FileInputStream fis = new FileInputStream(new File(path)); if (path.contains("yml")) { @@ -64,7 +64,7 @@ public class FileConfiguratorTest { LOGGER.info("修改文件完毕 准备触发监听器"); Thread.sleep(2000); LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getString("testKey")); - Thread.sleep(12000); + Thread.sleep(2000); LOGGER.info("移除监听器之前，testKey：{}",configurator.getString("testKey")); configurator.removeConfigListener("/bakapplication.yml"); LOGGER.info("移除监听器之后，testKey：{}",configurator.getString("testKey")); @@ -76,7 +76,7 @@ public class FileConfiguratorTest { configurator.addConfigListener("/bakapplication.yml"); modifyFile(path); LOGGER.info("修改文件完毕"); - Thread.sleep(12000); + Thread.sleep(1000); } -- Gitee From 4a278a841fdd7b1fe717dc90df01ac79eb1b0972 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Mon, 14 Mar 2022 08:22:39 +0800 Subject: [PATCH 19/22] =?UTF-8?q?=E7=AE=80=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/client/TracerClientStarter.java | 3 - .../jlog/client/filter/HttpFilter.java | 21 ++-- .../jlog/common/handler/CompressHandler.java | 17 ++- .../{TagHandler.java => ExtractHandler.java} | 37 +++---- .../jlog/common/handler}/JcProperties.java | 7 +- .../config/apollo/ApolloConfigurator.java | 64 ++--------- .../jd/platform/jlog/core/ClientHandler.java | 31 ++++++ ...Builder.java => ClientHandlerBuilder.java} | 8 +- .../jd/platform/jlog/core/Configurator.java | 39 ------- .../com/jd/platform/jlog/core/FileNode.java | 1 + .../platform/jlog/etcd/EtcdConfigurator.java | 62 +---------- .../jd/platform/jlog/etcd/EtcdListener.java | 10 +- .../jlog/nacos/NacosConfigurator.java | 104 ++---------------- .../jd/platform/jlog/nacos/NacosListener.java | 3 + .../jd/platform/jlog/zk/ZkConfigurator.java | 60 +--------- .../com/jd/platform/jlog/test/Common.java | 2 +- .../jlog/test/EtcdConfiguratorTest.java | 10 +- .../jlog/test/FileConfiguratorTest.java | 3 - .../jlog/test/NacosConfiguratorTest.java | 10 +- .../jlog/test/ZKConfiguratorTest.java | 12 +- 20 files changed, 115 insertions(+), 389 deletions(-) rename common/src/main/java/com/jd/platform/jlog/common/handler/{TagHandler.java => ExtractHandler.java} (88%) rename {config/config-core/src/main/java/com/jd/platform/jlog/core => common/src/main/java/com/jd/platform/jlog/common/handler}/JcProperties.java (97%) create mode 100644 config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java rename config/config-core/src/main/java/com/jd/platform/jlog/core/{HandlerBuilder.java => ClientHandlerBuilder.java} (81%) diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index 9c329a0..c03c60b 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -106,8 +106,5 @@ public class TracerClientStarter { LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig)); Configurator configurator = ConfiguratorFactory.getInstance(); HandlerBuilder.buildHandler(tagConfig, configurator); - configurator.addConfigListener("/application.properties"); - // configurator.addConfigListener("/application.yml"); - } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 5756a80..241866f 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -5,12 +5,10 @@ import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.handler.CompressHandler; -import com.jd.platform.jlog.common.handler.TagHandler; import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; -import com.jd.platform.jlog.common.utils.ZstdUtils; +import com.jd.platform.jlog.core.ClientHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +128,7 @@ public class HttpFilter implements Filter { byte[] contentBytes = mResp.getContent(); String content = new String(contentBytes); Map responseMap = new HashMap<>(8); - responseMap.put("response", CompressHandler.compressResp(contentBytes)); + responseMap.put("response", ClientHandler.compressResp(contentBytes)); tracerObject.add(responseMap); //此处可以对content做处理,然后再把content写回到输出流中 @@ -148,12 +146,15 @@ public class HttpFilter implements Filter { long tracerId, String uri) { //request的各个入参 Map params = servletRequest.getParameterMap(); - Map extMap = new HashMap<>(params.size()); - extMap.put("appName", Context.APP_NAME); - extMap.put("serverIp", IpUtils.getIp()); - extMap.put("tracerId", tracerId); - extMap.put("uri", uri); - tracerObject.add(TagHandler.extractReqTag(params, extMap)); + Map requestMap = new HashMap<>(params.size()); + for (String key : params.keySet()) { + requestMap.put(key, params.get(key)[0]); + } + requestMap.put("appName", Context.APP_NAME); + requestMap.put("serverIp", IpUtils.getIp()); + requestMap.put("tracerId", tracerId); + requestMap.put("uri", uri); + tracerObject.add(ClientHandler.processReq(requestMap); } @Override diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java index 82b70f4..20bf8ad 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -1,8 +1,11 @@ package com.jd.platform.jlog.common.handler; +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.utils.ZstdUtils; +import com.sun.istack.internal.NotNull; import java.util.Base64; +import java.util.Map; import static com.jd.platform.jlog.common.constant.Constant.MIN; import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD; @@ -46,12 +49,18 @@ public class CompressHandler { instance = new CompressHandler(compress, threshold); } - public static byte[] compressReq(byte[] contentBytes){ - if(instance == null || !isMatched(instance.compress, E_REQ)){ return contentBytes; } - return doCompress(contentBytes); + public static Map compressReq(Map map){ + if(instance == null || !isMatched(instance.compress, E_REQ)){ + return map; + } + + for (Map.Entry entry : map.entrySet()) { + map.put(entry.getKey(), doCompress(entry.getValue().toString().getBytes())); + } + return map; } - public static byte[] compressLog(byte[] contentBytes){ + public static Map compressLog(byte[] contentBytes){ if(instance == null || !isMatched(instance.compress, E_LOG)){ return contentBytes; } return doCompress(contentBytes); } diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java similarity index 88% rename from common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java index 0f4619a..9e2d790 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/TagHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java @@ -23,12 +23,12 @@ import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM; /** * @author tangbohu * @version 1.0.0 - * @ClassName Tag.java + * @ClassName ExtractHandler.java * @createTime 2022年02月12日 21:28:00 */ -public class TagHandler { +public class ExtractHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(TagHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ExtractHandler.class); private List reqTags; @@ -46,7 +46,7 @@ public class TagHandler { private long extract; - private static volatile TagHandler instance = null; + private static volatile ExtractHandler instance = null; /** * 构建标签处理器 @@ -58,7 +58,7 @@ public class TagHandler { return; } - TagHandler handler = new TagHandler(); + ExtractHandler handler = new ExtractHandler(); handler.extract = tagConfig.getExtract(); handler.reqTags = tagConfig.getReqTags(); handler.logTags = tagConfig.getLogTags(); @@ -83,32 +83,25 @@ public class TagHandler { /** * 提取请求参数里的标签 - * @param params 参数 - * @param ext 额外附加的,如ip等 + * @param reqMap 额外附加的,如ip等 * @return tags */ - public static Map extractReqTag(Map params, @NotNull Map ext) { + public static Map extractReqTag(Map reqMap) { if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); - System.out.println("### .params:"+JSON.toJSONString(params)); - System.out.println("### .ext:"+JSON.toJSONString(ext)); + System.out.println("### .ext:"+JSON.toJSONString(reqMap)); - Map requestMap = new HashMap<>(instance.reqTags.size()); + Map tagMap = new HashMap<>(instance.reqTags.size()); for (String tag : instance.reqTags) { - Object val = ext.get(tag); + Object val = reqMap.get(tag); if(val != null){ - requestMap.put(tag, val); - continue; - } - - if(CollectionUtil.isNotEmpty(params) && params.get(tag) != null){ - requestMap.put(tag, params.get(tag)[0]); + tagMap.put(tag, val); } } - System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(requestMap)); - return requestMap; + System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(tagMap)); + return tagMap; } @@ -155,9 +148,9 @@ public class TagHandler { if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } - System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); + System.out.println("### INSTANCE.respTags:"+JSON.toJSONString(instance.respTags)); - Map requestMap = new HashMap<>(instance.reqTags.size()); + Map requestMap = new HashMap<>(instance.respTags.size()); for (String tag : instance.reqTags) { Object val = resp.get(tag); if(val != null){ diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java similarity index 97% rename from config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java rename to common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java index b907677..8e4b838 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/JcProperties.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java @@ -1,4 +1,4 @@ -package com.jd.platform.jlog.core; +package com.jd.platform.jlog.common.handler; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.utils.ConfigUtil; @@ -8,7 +8,10 @@ import com.jd.platform.jlog.common.utils.StringUtil; import java.lang.reflect.Field; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst; diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index f0e6432..d671394 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -36,7 +36,6 @@ public class ApolloConfigurator implements Configurator { * 里面有resourceProperties 和 configProperties */ private static volatile Config config; - private static final ConcurrentMap CONFIG_LISTENER_MAP = new ConcurrentHashMap<>(); private static volatile ApolloConfigurator instance; private ApolloConfigurator() { @@ -49,14 +48,10 @@ public class ApolloConfigurator implements Configurator { config.addChangeListener(changeEvent -> { LOGGER.info("Apollo收到事件变更, keys={}", changeEvent.changedKeys()); for (String key : changeEvent.changedKeys()) { - if (!CONFIG_LISTENER_MAP.containsKey(key)) { - LOGGER.info("{}不是关注的key,直接返回", key); - continue; - } ConfigChange change = changeEvent.getChange(key); ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType())); - CONFIG_LISTENER_MAP.get(key).onProcessEvent(event); + new ApolloListener().onProcessEvent(event); } }); } @@ -97,17 +92,6 @@ public class ApolloConfigurator implements Configurator { return null; } - @Override - public String getConfig(String key) { - return config.getProperty(key,""); - } - - @Override - public String getConfig(String key, long timeoutMills) { - return getConfig(key,1L); - } - - @Override public boolean putConfig(String key, String content) { return false; @@ -120,46 +104,12 @@ public class ApolloConfigurator implements Configurator { } - - @Override - public boolean removeConfig(String dataId, long timeoutMills) { - return false; - } - - - @Override - public boolean removeConfig(String key) { - return false; - } - - @Override - public void addConfigListener(String key) { - if (StringUtils.isBlank(key)) { - return; - } - CONFIG_LISTENER_MAP.put(key, new ApolloListener()); - } - - - @Override - public void removeConfigListener(String key) { - System.out.println("Apollo进入移除"+key); - CONFIG_LISTENER_MAP.remove(key); - } - - @Override public String getType() { return "apollo"; } - @Override - public List getConfigByPrefix(String prefix) { - return null; - } - - private ConfigChangeType getChangeType(PropertyChangeType changeType) { switch (changeType) { case ADDED: @@ -176,24 +126,24 @@ public class ApolloConfigurator implements Configurator { Properties properties = System.getProperties(); if (!properties.containsKey(PROP_APP_ID)) { - System.setProperty(PROP_APP_ID, FILE_CONFIG.getConfig(APP_ID)); + System.setProperty(PROP_APP_ID, FILE_CONFIG.getString(APP_ID)); } if (!properties.containsKey(PROP_APOLLO_META)) { - System.setProperty(PROP_APOLLO_META, FILE_CONFIG.getConfig(APOLLO_META)); + System.setProperty(PROP_APOLLO_META, FILE_CONFIG.getString(APOLLO_META)); } if (!properties.containsKey(PROP_APOLLO_SECRET)) { - String secretKey = FILE_CONFIG.getConfig(APOLLO_SECRET); + String secretKey = FILE_CONFIG.getString(APOLLO_SECRET); if (!StringUtils.isBlank(secretKey)) { System.setProperty(PROP_APOLLO_SECRET, secretKey); } } if (!properties.containsKey(APOLLO_CLUSTER)) { - if (!StringUtils.isBlank(FILE_CONFIG.getConfig(APOLLO_CLUSTER))) { - System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getConfig(APOLLO_CLUSTER)); + if (!StringUtils.isBlank(FILE_CONFIG.getString(APOLLO_CLUSTER))) { + System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getString(APOLLO_CLUSTER)); } } if (!properties.containsKey(APOLLO_CONFIG_SERVICE)) { - System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getConfig(APOLLO_CONFIG_SERVICE)); + System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getString(APOLLO_CONFIG_SERVICE)); } } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java new file mode 100644 index 0000000..a087697 --- /dev/null +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java @@ -0,0 +1,31 @@ +package com.jd.platform.jlog.core; + +import com.jd.platform.jlog.common.handler.CompressHandler; +import com.jd.platform.jlog.common.handler.ExtractHandler; +import java.util.Map; + +/** + * @author tangbohu + * @version 1.0.0 + * @ClassName ClientHandler.java + * @createTime 2022年03月13日 16:53:00 + */ +public class ClientHandler { + + public static Map processReq(Map reqMap){ + Map map = ExtractHandler.extractReqTag(reqMap); + return CompressHandler.compressReq(map);; + } + + + public static Map processLog(){ + + return null; + } + + + public static Map processResp(byte[] resp, String content){ + byte[] contentBytes + return 1; + } +} diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java similarity index 81% rename from config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java rename to config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java index 5d13796..c98663d 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/HandlerBuilder.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.core; import com.jd.platform.jlog.common.handler.CompressHandler; +import com.jd.platform.jlog.common.handler.ExtractHandler; import com.jd.platform.jlog.common.handler.TagConfig; -import com.jd.platform.jlog.common.handler.TagHandler; /** * @author tangbohu @@ -11,7 +11,7 @@ import com.jd.platform.jlog.common.handler.TagHandler; * @Description TODO * @createTime 2022年03月05日 22:07:00 */ -public class HandlerBuilder { +public class ClientHandlerBuilder { @@ -19,7 +19,7 @@ public class HandlerBuilder { if(tagConfig == null){ tagConfig = buildTagConfigByConfigurator(configurator); } - TagHandler.buildTagHandler(tagConfig); + ExtractHandler.buildTagHandler(tagConfig); CompressHandler.buildCompressHandler(configurator.getLong("compress"), configurator.getLong("threshold")); } @@ -29,7 +29,7 @@ public class HandlerBuilder { if(configurator == null){ throw new RuntimeException("configurator is null"); } - TagHandler.refresh(buildTagConfigByConfigurator(configurator)); + ExtractHandler.refresh(buildTagConfigByConfigurator(configurator)); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java index fd25c7b..ad4bdf4 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java @@ -59,45 +59,6 @@ public interface Configurator { boolean putConfig(String key, String content, long timeoutMills); - /** - * 移除配置 - * @param key key - * @return val - */ - boolean removeConfig(String key); - - /** - * 移除配置 - * @param key key - * @param timeoutMills timeoutMills - * @return val - */ - boolean removeConfig(String key, long timeoutMills); - - - /** - * 根据前缀/父级路径获取子节点 - * @param prefix path - * @return List - */ - List getConfigByPrefix(String prefix); - - - /** - * 添加监听器 - * @param node 文件 - */ - void addConfigListener(String node); - - - /** - * 移除监听器 - * @param node 节点 file or dir - */ - @Deprecated - void removeConfigListener(String node); - - /** * 获取配置器类型 * @return string example:apollo diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java index 9756593..44b3c99 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.handler.CollectMode; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.handler.TagConfig; import com.jd.platform.jlog.common.utils.ZstdUtils; diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index 060fe06..914c749 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -40,17 +40,15 @@ public class EtcdConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + public static final String ROOT = "/jLog"; private static final String PROPERTIES_PATH = "/jLog/jLog.properties"; private static Properties PROPERTIES = new Properties(); - private volatile static EtcdListener LISTENER = null; - - private EtcdConfigurator() { - LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getConfig(SERVER_ADDR_KEY)); - client = EtcdClient.forEndpoints(FILE_CONFIG.getConfig(SERVER_ADDR_KEY,2000L)).withPlainText().build(); + LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getString(SERVER_ADDR_KEY)); + client = EtcdClient.forEndpoints(FILE_CONFIG.getString(SERVER_ADDR_KEY)).withPlainText().build(); RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync(); List keyValues = rangeResponse.getKvsList(); if (CollectionUtil.isEmpty(keyValues)) { @@ -64,6 +62,8 @@ public class EtcdConfigurator implements Configurator { e.printStackTrace(); } } + + new EtcdListener().onProcessEvent(new ConfigChangeEvent()); } @@ -99,20 +99,6 @@ public class EtcdConfigurator implements Configurator { return null; } - @Override - public String getConfig(String key) { - Object val = PROPERTIES.get(key); - if(val != null){ - return String.valueOf(val); - } - return null; - } - - @Override - public String getConfig(String key, long timeoutMills) { - return getConfig(key); - } - @Override public boolean putConfig(String key, String content) { return putConfig(key, content, DEFAULT_TIMEOUT); @@ -128,48 +114,10 @@ public class EtcdConfigurator implements Configurator { return true; } - @Override - public boolean removeConfig(String key) { - return removeConfig(key, DEFAULT_TIMEOUT); - } - - - @Override - public boolean removeConfig(String key, long timeoutMills) { - PROPERTIES.remove(key); - client.getKvClient().put(ByteString.copyFromUtf8(PROPERTIES_PATH), ByteString.copyFromUtf8(formatConfigStr(PROPERTIES))).sync(); - return true; - } - - @Override - public void addConfigListener(String node) { - System.out.println("添加etcd监听器"+node); - LISTENER = new EtcdListener(node); - LISTENER.onProcessEvent(new ConfigChangeEvent()); - } - - @Override - public void removeConfigListener(String node) { - System.out.println("移除etcd监听器"+node); - LISTENER.onShutDown(); - LISTENER = null; - } @Override public String getType() { return "etcd"; } - - @Override - public List getConfigByPrefix(String prefix) { - RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(prefix)).asPrefix().sync(); - List keyValues = rangeResponse.getKvsList(); - List list = new ArrayList<>(); - for (KeyValue kv : keyValues) { - list.add(kv.getValue().toStringUtf8()); - } - return list; - } - } diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index 53bf0e9..c10b1b2 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -16,6 +16,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static com.jd.platform.jlog.etcd.EtcdConfigurator.ROOT; + /** * @author tangbohu * @version 1.0.0 @@ -29,11 +31,9 @@ public class EtcdListener implements ConfigChangeListener { TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("etcdListener", 1)); - public EtcdListener(String node) { - - iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(node)).asPrefix().start(); - System.out.println("构造器EtcdListener"+node); + public EtcdListener() { + iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(ROOT)).asPrefix().start(); getExecutorService().submit(() -> { while (iterator.hasNext()){ try { @@ -43,7 +43,7 @@ public class EtcdListener implements ConfigChangeListener { Event.EventType eveType = eve.getType(); ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE; ConfigChangeEvent event = new ConfigChangeEvent(); - event.setKey(node).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); + event.setKey(kv.getKey().toStringUtf8()).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType); onChangeEvent(event); }catch (RuntimeException e){ e.printStackTrace(); diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java index 88212aa..dad6364 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -25,6 +25,9 @@ import static com.jd.platform.jlog.nacos.NacosConstant.*; +/** + * @author tangbohu + */ public class NacosConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); @@ -71,6 +74,8 @@ public class NacosConfigurator implements Configurator { KEY_DATAID_MAP.put((String) e.nextElement(), DEFAULT_DATA_ID); } } + configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, new NacosListener()); + } catch (NacosException | IOException e) { throw new RuntimeException(e); } @@ -98,39 +103,6 @@ public class NacosConfigurator implements Configurator { return null; } - @Override - public String getConfig(String key) { - return getConfig(key, DEFAULT_TIMEOUT); - } - - - @Override - public String getConfig(String key, long timeoutMills) { - String value = System.getProperty(key); - if (value != null) { - return value; - } - - value = PROPERTIES.getProperty(key); - - if (null == value) { - try { - String dataId = KEY_DATAID_MAP.get(key); - if(StringUtil.isEmpty(dataId)){ - return null; - } - String config = configService.getConfig(dataId, JLOG_GROUP, timeoutMills); - if(StringUtil.isEmpty(config)){ - return null; - } - PROPERTIES.load(new StringReader(config)); - } catch (NacosException | IOException ex) { - LOGGER.error(ex.getMessage()); - } - } - return PROPERTIES.getProperty(key); - } - @Override public boolean putConfig(String key, String content) { @@ -159,71 +131,15 @@ public class NacosConfigurator implements Configurator { } - @Override - public boolean removeConfig(String key) { - return removeConfig(key, DEFAULT_TIMEOUT); - } - - - - @Override - public boolean removeConfig(String key, long timeoutMills) { - - String dataId = KEY_DATAID_MAP.get(key); - if(StringUtil.isEmpty(dataId)){ - return false; - } - boolean result = false; - try { - if (!PROPERTIES.isEmpty()) { - PROPERTIES.remove(key); - result = configService.publishConfig(dataId, JLOG_GROUP, formatConfigStr(PROPERTIES)); - } else { - result = configService.removeConfig(dataId, JLOG_GROUP); - } - } catch (NacosException exx) { - LOGGER.error(exx.getErrMsg()); - } - return result; - } - - - - @Override - public void addConfigListener(String dataId) { - if(!DEFAULT_DATA_ID.equals(dataId)){ - throw new RuntimeException("no support"); - } - LOGGER.info("nacos添加监听器开始 dataId:{}",dataId); - NacosListener nacosListener = new NacosListener(); - LOGGER.info("## nacos添加监听器过程 {}",nacosListener.toString()); - try { - configService.addListener(dataId, JLOG_GROUP, nacosListener); - } catch (NacosException e) { - LOGGER.error("nacos添加监听器失败",e); - } - } - - - - @Override - public void removeConfigListener(String dataId) { - if(!DEFAULT_DATA_ID.equals(dataId)){ - throw new RuntimeException("no support"); - } - LOGGER.info("nacos移除监听器开始"); - configService.removeListener(dataId, JLOG_GROUP, NACOSLISTENER); - } - private static Properties getConfigProperties() { Properties properties = new Properties(); - String address = FILE_CONFIG.getConfig(SERVER_ADDR_KEY); + String address = FILE_CONFIG.getString(SERVER_ADDR_KEY); if (address != null) { properties.setProperty(SERVER_ADDR_KEY, address); } - String namespace = FILE_CONFIG.getConfig(NAMESPACE_KEY); + String namespace = FILE_CONFIG.getString(NAMESPACE_KEY); if (namespace != null) { properties.setProperty(NAMESPACE_KEY, namespace); }else{ @@ -242,10 +158,4 @@ public class NacosConfigurator implements Configurator { } - @Override - public List getConfigByPrefix(String prefix) { - return null; - } - - } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java index da7d2c7..06ed9f4 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -19,6 +19,9 @@ import static com.jd.platform.jlog.nacos.NacosConfigurator.*; import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; +/** + * @author tangbohu + */ public class NacosListener extends AbstractSharedListener implements ConfigChangeListener, EventListener { diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java index c1130bb..a6bd0db 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -57,7 +57,7 @@ public class ZkConfigurator implements Configurator { zkClient.create().withMode(CreateMode.EPHEMERAL).forPath(DEFAULT_WORKER_PATH); } loadZkData(); - addConfigListener(DEFAULT_CONFIG_PATH); + new ZkListener(DEFAULT_CONFIG_PATH); LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(PROPERTIES)); } } @@ -106,64 +106,6 @@ public class ZkConfigurator implements Configurator { - @Override - public boolean removeConfig(String key) { - return removeConfig(key, DEFAULT_TIMEOUT); - } - - - - @Override - public boolean removeConfig(String key, long timeoutMills) { - PROPERTIES.remove(key); - try { - zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigByte(PROPERTIES)); - } catch (Exception e) { - return false; - } - return true; - } - - - - - @Override - public void addConfigListener(String node) { - if(!DEFAULT_CONFIG_PATH.equals(node)){ - throw new RuntimeException("no support"); - } - LOGGER.info("ZK添加监听器, node:{}", node); - if(ZKLISTENER == null){ - synchronized (ZkConfigurator.class){ - ZKLISTENER = new ZkListener(node); - } - } - } - - - @Override - public void removeConfigListener(String node) { - if(!DEFAULT_CONFIG_PATH.equals(node)){ - throw new RuntimeException("no support"); - } - LOGGER.info("ZK删除监听器, node:{}", node); - ZKLISTENER.onShutDown(); - ZKLISTENER = null; - } - - - @Override - public List getConfigByPrefix(String prefix) { - try { - String val = getString(prefix); - return FastJsonUtils.toList(val,String.class); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - @Override public String getType() { return "zk"; diff --git a/example/src/test/java/com/jd/platform/jlog/test/Common.java b/example/src/test/java/com/jd/platform/jlog/test/Common.java index 24d6e04..3f10c42 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/Common.java +++ b/example/src/test/java/com/jd/platform/jlog/test/Common.java @@ -29,7 +29,7 @@ public class Common { LOGGER.info("配置器get addr：{}", addr); TagConfig tagConfig = configurator.getObject("tagConfig", TagConfig.class); LOGGER.info("配置器get tagConfig：{}", tagConfig.toString()); - List workers = configurator.getConfigByPrefix("workers"); + List workers = configurator.getList("workers"); LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); } diff --git a/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java index 2945261..4a5b575 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java @@ -45,7 +45,7 @@ public class EtcdConfiguratorTest { // @Test public void testUpdateCFG() throws Exception { - List workers = configurator.getConfigByPrefix("workers"); + List workers = configurator.getList("workers"); LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); String myIp = "121.1.1.0"; if(workers.contains(myIp)){ @@ -57,7 +57,7 @@ public class EtcdConfiguratorTest { workers.add(myIp); } configurator.putConfig("workers",JSON.toJSONString(workers)); - List workers2 = configurator.getConfigByPrefix("workers"); + List workers2 = configurator.getList("workers"); LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); } @@ -68,7 +68,6 @@ public class EtcdConfiguratorTest { String val1 = configurator.getString("testKey"); LOGGER.info("初始化的testKey的val:{}", val1); - configurator.addConfigListener("/jLog"); LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); configurator.putConfig("testKey",i1 + ""); LOGGER.info("修改完毕 准备触发监听器"); @@ -76,14 +75,9 @@ public class EtcdConfiguratorTest { String val2 = configurator.getString("testKey"); LOGGER.info("第一次修改后的的val:{}", val2); Thread.sleep(5000); - configurator.removeConfigListener("/jLog"); - Thread.sleep(5000); // LOGGER.info("移除监听器后:修改配置testKey = {}",i2); // configurator.putConfig("testKey",i2 + ""); LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); - - - configurator.addConfigListener("/jLog"); LOGGER.info("第二次添加监听器"); Thread.sleep(22000); diff --git a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java index 88e8267..52e7456 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -50,7 +50,6 @@ public class FileConfiguratorTest { @Test public void testAddConfigListener() throws Exception { - configurator.addConfigListener("/bakapplication.yml"); String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; Properties props = new Properties(); FileInputStream fis = new FileInputStream(new File(path)); @@ -66,14 +65,12 @@ public class FileConfiguratorTest { LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getString("testKey")); Thread.sleep(2000); LOGGER.info("移除监听器之前，testKey：{}",configurator.getString("testKey")); - configurator.removeConfigListener("/bakapplication.yml"); LOGGER.info("移除监听器之后，testKey：{}",configurator.getString("testKey")); modifyFile(path); LOGGER.info("修改文件完毕 准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); LOGGER.info("再次添加监听器"); - configurator.addConfigListener("/bakapplication.yml"); modifyFile(path); LOGGER.info("修改文件完毕"); Thread.sleep(1000); diff --git a/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java index bcc1be6..38c77c9 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java @@ -45,7 +45,7 @@ public class NacosConfiguratorTest { @Test public void testUpdateCFG() throws Exception { - List workers = configurator.getConfigByPrefix("workers"); + List workers = configurator.getList("workers"); LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); String myIp = "121.1.1.0"; if(workers.contains(myIp)){ @@ -57,7 +57,7 @@ public class NacosConfiguratorTest { workers.add(myIp); } configurator.putConfig("workers",JSON.toJSONString(workers)); - List workers2 = configurator.getConfigByPrefix("workers"); + List workers2 = configurator.getList("workers"); LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); } @@ -68,17 +68,11 @@ public class NacosConfiguratorTest { String val1 = configurator.getString("testKey"); LOGGER.info("初始化的testKey的val:{}", val1); - configurator.addConfigListener("bakjLog.properties"); - LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); // configurator.putConfig("testKey",i1 + ""); LOGGER.info("修改完毕 准备触发监听器"); Thread.sleep(1000); String val2 = configurator.getString("testKey"); LOGGER.info("第一次修改后的的val:{}", val2); - Thread.sleep(1000); - configurator.removeConfigListener("bakjLog.properties"); - Thread.sleep(1000); - LOGGER.info("移除监听器后:修改配置testKey = {}",i2); configurator.putConfig("testKey",i2 + ""); LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); diff --git a/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java index 4f320dd..bbd1788 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java @@ -46,7 +46,7 @@ public class ZKConfiguratorTest { @Test public void testUpdateCFG() throws Exception { - List workers = configurator.getConfigByPrefix("workers"); + List workers = configurator.getList("workers"); LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); String myIp = "121.1.1.0"; if(workers.contains(myIp)){ @@ -58,7 +58,7 @@ public class ZKConfiguratorTest { workers.add(myIp); } configurator.putConfig("workers",JSON.toJSONString(workers)); - List workers2 = configurator.getConfigByPrefix("workers"); + List workers2 = configurator.getList("workers"); LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2)); } @@ -69,7 +69,6 @@ public class ZKConfiguratorTest { String val1 = configurator.getString("testKey"); LOGGER.info("初始化的testKey的val:{}", val1); - configurator.addConfigListener("/bakjLog.properties"); LOGGER.info("添加监听器后， 修改配置testKey = {}", i1); // configurator.putConfig("testKey",i1 + ""); LOGGER.info("修改完毕 准备触发监听器"); @@ -77,12 +76,5 @@ public class ZKConfiguratorTest { String val2 = configurator.getString("testKey"); LOGGER.info("第一次修改后的的val:{}", val2); Thread.sleep(1000); - configurator.removeConfigListener("/bakjLog.properties"); - Thread.sleep(1000); - LOGGER.info("移除监听器后:修改配置testKey = {}",i2); - // configurator.putConfig("testKey",i2 + ""); - LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); - - Thread.sleep(35000); } } -- Gitee From 2e41277edf7110768a9e475bd2423841a5900fe6 Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Mon, 14 Mar 2022 20:38:09 +0800 Subject: [PATCH 20/22] =?UTF-8?q?=E5=85=A8=E9=9D=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 10 +- .../jlog/client/TracerClientStarter.java | 4 +- .../jlog/client/filter/HttpFilter.java | 10 +- .../platform/jlog/client/jsf/JsfRequest.java | 41 ----- .../platform/jlog/client/jsf/JsfTracer.java | 91 ---------- .../jd/platform/jlog/client/task/Monitor.java | 16 +- .../jlog/common/handler/CollectMode.java | 4 - .../jlog/common/handler/CompressHandler.java | 9 +- .../jlog/common/handler/ExtractHandler.java | 19 +-- .../jlog/common/handler/JcProperties.java | 114 +++---------- .../jlog/common/utils/ConfigUtil.java | 87 ++++++++++ .../config/apollo/ApolloConfigurator.java | 34 +++- .../jlog/config/apollo/ApolloConstant.java | 1 - .../jlog/config/apollo/ApolloListener.java | 3 +- .../jd/platform/jlog/core/ClientHandler.java | 46 ++++- .../jlog/core/ClientHandlerBuilder.java | 9 +- .../platform/jlog/core/ConfigChangeEvent.java | 3 +- .../jlog/core/ConfiguratorFactory.java | 20 +-- .../com/jd/platform/jlog/core/Constant.java | 9 +- .../platform/jlog/core/FileConfigurator.java | 103 ++++------- .../platform/jlog/etcd/EtcdConfigurator.java | 18 +- .../jd/platform/jlog/etcd/EtcdListener.java | 45 ++++- .../jlog/nacos/NacosConfigurator.java | 46 ++--- .../jd/platform/jlog/nacos/NacosConstant.java | 15 -- .../jd/platform/jlog/nacos/NacosListener.java | 27 +-- .../jlog/nacos => test}/NacosTest.java | 0 .../jd/platform/jlog/zk/ZkConfigurator.java | 25 +-- .../jlog/zk/ZkConfiguratorProvider.java | 3 + .../com/jd/platform/jlog/zk/ZkConstant.java | 15 -- .../com/jd/platform/jlog/zk/ZkListener.java | 27 +-- .../java/com/jd/platform/jlog/zk/ZkTest.java | 60 ------- .../appender/TracerLogbackAppender.java | 7 +- .../src/main/resources/application.properties | 21 +-- example/src/main/resources/application.ymlbak | 28 --- example/src/main/resources/bakapplication.yml | 19 ++- example/src/main/resources/bakjLog.properties | 23 ++- .../com/jd/platform/jlog/test/Common.java | 2 +- .../jlog/test/FileConfiguratorTest.java | 17 +- .../jlog/test/ZKConfiguratorTest.java | 3 +- .../jlog/worker/config/CenterStarter.java | 20 +-- .../jlog/worker/config/ConfigCenter.java | 32 ---- .../jlog/worker/disruptor/TracerConsumer.java | 57 +------ .../jlog/worker/entity/TracerModel.java | 160 ------------------ 43 files changed, 407 insertions(+), 896 deletions(-) delete mode 100644 client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java delete mode 100644 client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java delete mode 100644 config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java rename config/config-nacos/src/{main/java/com/jd/platform/jlog/nacos => test}/NacosTest.java (100%) delete mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java delete mode 100644 config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java delete mode 100644 example/src/main/resources/application.ymlbak delete mode 100644 worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java delete mode 100644 worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java diff --git a/client/pom.xml b/client/pom.xml index 6de7c96..47d1a0e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -23,7 +23,7 @@ config-zk 1.4-SNAPSHOT --> - - - + + javax.servlet servlet-api diff --git a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java index c03c60b..32f41c3 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java @@ -5,9 +5,9 @@ import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.task.Monitor; import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.core.ClientHandlerBuilder; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; -import com.jd.platform.jlog.core.HandlerBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,6 +105,6 @@ public class TracerClientStarter { private void initJLogConfig(){ LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig)); Configurator configurator = ConfiguratorFactory.getInstance(); - HandlerBuilder.buildHandler(tagConfig, configurator); + ClientHandlerBuilder.buildHandler(tagConfig, configurator); } } diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 241866f..2fd16ba 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -128,7 +128,13 @@ public class HttpFilter implements Filter { byte[] contentBytes = mResp.getContent(); String content = new String(contentBytes); Map responseMap = new HashMap<>(8); - responseMap.put("response", ClientHandler.compressResp(contentBytes)); + Map map = new HashMap<>(1); + map.put("errno", 200); + ClientHandler.Outcome outcome = ClientHandler.processResp(contentBytes, map); + responseMap.put("response", outcome.getContent()); + if(!outcome.getMap().isEmpty()){ + responseMap.putAll(outcome.getMap()); + } tracerObject.add(responseMap); //此处可以对content做处理,然后再把content写回到输出流中 @@ -154,7 +160,7 @@ public class HttpFilter implements Filter { requestMap.put("serverIp", IpUtils.getIp()); requestMap.put("tracerId", tracerId); requestMap.put("uri", uri); - tracerObject.add(ClientHandler.processReq(requestMap); + tracerObject.add(ClientHandler.processReq(requestMap)); } @Override diff --git a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java b/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java deleted file mode 100644 index 27e2559..0000000 --- a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.jd.platform.jlog.client.jsf; - - -/** - * jsf请求入参包装对象 - * @author wuweifeng - * @version 1.0 - * @date 2021-10-26 - */ -public interface JsfRequest { - /** - * 请求的完整入参 - */ - String requestContent(); - - /** - * 接口名 - */ - String uri(); - - /** - * 用户pin - */ - default String pin() { - return ""; - } - - /** - * 用户uuid - */ - default String uuid() { - return ""; - } - - /** - * 用户ip - */ - default String userIp() { - return ""; - } -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java b/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java deleted file mode 100644 index faa0dc9..0000000 --- a/client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.jd.platform.jlog.client.jsf; - -import com.jd.platform.jlog.client.Context; -import com.jd.platform.jlog.client.tracerholder.TracerHolder; -import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.model.TracerBean; -import com.jd.platform.jlog.common.utils.IdWorker; -import com.jd.platform.jlog.common.utils.IpUtils; -import com.jd.platform.jlog.common.utils.ZstdUtils; - -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** - * JsfTracer工具类 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-10-26 - */ -public class JsfTracer { - - /** - * 临时保存入参 - */ - private static Map TEMP_HOLDER = new HashMap<>(128); - - /** - * 跟踪开启 - */ - public static void begin(JsfRequest jsfRequest) { - long tracerId = IdWorker.nextId(); - TracerHolder.setTracerId(tracerId); - - TracerBean tracerBean = new TracerBean(); - tracerBean.setCreateTime(System.currentTimeMillis()); - - List> tracerObject = new ArrayList<>(); - tracerBean.setTracerObject(tracerObject); - - //将request信息保存 - Map requestMap = new HashMap<>(); - requestMap.put("appName", Context.APP_NAME); - requestMap.put("serverIp", IpUtils.getIp()); - - requestMap.put("tracerId", tracerId); - requestMap.put("uri", jsfRequest.uri().trim()); - //将用户整个request都放进去 - requestMap.put("wholeRequest", jsfRequest.requestContent()); - - requestMap.put("pin", jsfRequest.pin()); - requestMap.put("uuid", jsfRequest.uuid()); - requestMap.put("userIp", jsfRequest.userIp()); - - tracerObject.add(requestMap); - - tracerBean.setTracerId(tracerId + ""); - - TEMP_HOLDER.put(tracerId, tracerBean); - } - - /** - * 记录请求出入参，发送到worker - */ - public static void end(String response) { - long tracerId = TracerHolder.getTracerId(); - TracerBean tracerBean = TEMP_HOLDER.get(tracerId); - if (tracerBean == null) { - return; - } - - if (response != null) { - //最终的要发往worker的response，经历了base64压缩 - byte[] bytes = ZstdUtils.compress(response.getBytes(StandardCharsets.UTF_8)); - byte[] base64Bytes = Base64.getEncoder().encode(bytes); - Map responseMap = new HashMap<>(8); - responseMap.put("response", base64Bytes); - - List> tracerObject = tracerBean.getTracerObject(); - tracerObject.add(responseMap); - } - - //设置耗时 - tracerBean.setCostTime((int) (System.currentTimeMillis() - tracerBean.getCreateTime())); - UdpSender.offerBean(tracerBean); - - //从缓存删除它 - TEMP_HOLDER.remove(tracerId); - } - -} diff --git a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java index 92c4307..991a738 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java +++ b/client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java @@ -1,18 +1,13 @@ package com.jd.platform.jlog.client.task; -import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.mdc.Mdc; import com.jd.platform.jlog.client.worker.WorkerInfoHolder; -import com.jd.platform.jlog.common.constant.Constant; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -59,17 +54,10 @@ public class Monitor { private void fetch() throws Exception { Configurator configurator = ConfiguratorFactory.getInstance(); //获取所有worker的ip - List addresses = new ArrayList(); + List addresses; try { //如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有 - if (Context.MDC != null) { - String mdc = parseMdc(Context.MDC); - addresses = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc); - } - if (addresses == null || addresses.size() == 0) { - addresses = configurator.getConfigByPrefix(Constant.WORKER_PATH + Context.APP_NAME); - } - + addresses = configurator.getList("workers"); //全是空，给个警告 if (addresses == null || addresses.size() == 0) { LOGGER.warn("very important warn !!! workers ip info is null!!!"); diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java index 0140478..da4d954 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java @@ -93,12 +93,8 @@ public class CollectMode { public static final long COMPRESS_ALL = C_REQ | C_LOG | C_RESP; - public static void main(String[] args) { - System.out.println(isMatched(EXTRACT_REQ_LOG, EXTRACT_REQ_RESP)); - } public static boolean isMatched(long indicator, long position) { return (indicator & position) == position; } } - // curl --location --request POST 'http://10.96.98.110:8058/app/flow/getappraiselist' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'token=DF9YcBVAaMiLCZtZEZCf6MunJX8I3jWGmQXwRzb-oQIszD1uAjEQgNG7fG1Gq5nx2I6nzBFygwSWn8ZIIKrV3h0hUb3ubUwlKYsuijCNNGE6WVVLCLOQ1uvoLTza6KrCDPJNJfn5RfgjQfgnw8vo7j6KqX-HC0dyCCu58bg974f1U-_CibQWrXlXC-FM8lWrqhZrvVYvCBcSQ7iSur8CAAD__w==' --data-urlencode 'cityID=55000116' --data-urlencode 'appVersion=2.0.28' --data-urlencode 'country=BR' \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java index 20bf8ad..5d13f83 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java @@ -36,6 +36,7 @@ public class CompressHandler { this.threshold = threshold; } + private static volatile CompressHandler instance = null; @@ -49,6 +50,11 @@ public class CompressHandler { instance = new CompressHandler(compress, threshold); } + public static void refresh(Long compress, Long threshold){ + instance = null; + buildCompressHandler(compress, threshold); + } + public static Map compressReq(Map map){ if(instance == null || !isMatched(instance.compress, E_REQ)){ return map; @@ -60,7 +66,7 @@ public class CompressHandler { return map; } - public static Map compressLog(byte[] contentBytes){ + public static byte[] compressLog(byte[] contentBytes){ if(instance == null || !isMatched(instance.compress, E_LOG)){ return contentBytes; } return doCompress(contentBytes); } @@ -88,5 +94,4 @@ public class CompressHandler { this.compress = compress; } - } diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java index 9e2d790..b722076 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java @@ -52,8 +52,7 @@ public class ExtractHandler { * 构建标签处理器 * @param tagConfig 配置类 */ - public static void buildTagHandler(TagConfig tagConfig) { - + public static void buildExtractHandler(TagConfig tagConfig) { if(tagConfig.getExtract() == SUSPEND){ return; } @@ -76,7 +75,6 @@ public class ExtractHandler { handler.delimiterLen = tagConfig.getDelimiter().length(); handler.join = tagConfig.getJoin(); instance = handler; - System.out.println("### instance == > "+instance.toString()); LOGGER.info("构建标签处理器单例完成:{}",instance.toString()); } @@ -90,8 +88,7 @@ public class ExtractHandler { if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } - System.out.println("### INSTANCE.reqTags:"+JSON.toJSONString(instance.reqTags)); - System.out.println("### .ext:"+JSON.toJSONString(reqMap)); + System.out.println("### REQ INSTANCE :"+instance.toString()); Map tagMap = new HashMap<>(instance.reqTags.size()); for (String tag : instance.reqTags) { @@ -114,8 +111,6 @@ public class ExtractHandler { if(instance == null || !isMatched(instance.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){ return null; } - System.out.println("### INSTANCE:"+instance.toString()); - System.out.println("### .content:"+content); Map tagMap = new HashMap<>(3); Matcher m = instance.pattern.matcher(content); @@ -148,16 +143,14 @@ public class ExtractHandler { if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; } - System.out.println("### INSTANCE.respTags:"+JSON.toJSONString(instance.respTags)); - Map requestMap = new HashMap<>(instance.respTags.size()); - for (String tag : instance.reqTags) { + for (String tag : instance.respTags) { Object val = resp.get(tag); if(val != null){ requestMap.put(tag, val); } } - System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(requestMap)); + System.out.println("提取到了请求出参日志标签："+JSON.toJSONString(requestMap)); return requestMap; } @@ -170,7 +163,7 @@ public class ExtractHandler { */ public synchronized static void refresh(TagConfig tagConfig) { instance = null; - buildTagHandler(tagConfig); + buildExtractHandler(tagConfig); } @@ -194,7 +187,7 @@ public class ExtractHandler { cfg.setDelimiter("|"); cfg.setJoin("="); - buildTagHandler(cfg); + buildExtractHandler(cfg); List list = new ArrayList<>(); // Matcher m = BRACKET_PATTERN.matcher(content); diff --git a/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java index 8e4b838..4a8e9ab 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java +++ b/common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java @@ -13,6 +13,7 @@ import java.util.Date; import java.util.List; import java.util.Properties; +import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke; import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst; /** @@ -42,14 +43,35 @@ public class JcProperties extends Properties { return Long.parseLong(val); } + public List getStrList(String key) { - String val = getString(key); - if(StringUtil.isEmpty(val)){ + if(StringUtil.isEmpty(key)){ return null; } - return FastJsonUtils.toList(val, String.class); + String val = getString(key); + if(StringUtil.isNotEmpty(val)){ + return FastJsonUtils.toList(val,String.class); + } + return null; } + /* public List getStrList(String key) { + if(StringUtil.isEmpty(key)){ + return null; + } + int index = 0; + String suffix; + String fastSuffix; + List list = new ArrayList<>(); + do{ + suffix = "["+index+"]"; + fastSuffix = "["+(index+1)+"]"; + list.add(getString(key+suffix)); + index ++; + }while (getString(key + fastSuffix) != null); + return list; + }*/ + public T getBean(String key, Class clz) { T bean = FastJsonUtils.toBean(JSON.toJSONString(get(key)), clz); if(bean != null){ @@ -67,91 +89,5 @@ public class JcProperties extends Properties { } - /** - * 只支持简单的对象形配置 - * @param model bean - * @param properties - * @param prefix - * @throws IllegalAccessException - * @throws ClassNotFoundException - * @throws InstantiationException - * @throws ParseException - */ - @Deprecated - private void invoke(Object model, JcProperties properties, String prefix) throws - IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException { - - Class clz = model.getClass(); - Field[] fields = model.getClass().getDeclaredFields(); - for (Field field : fields) { - String type = field.getGenericType().toString(); - field.setAccessible(true); - - String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName())); - - prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix; - String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName(); - - - switch (type){ - case "class java.lang.String": - field.set(model, properties.getString(fillName)) ; - break; - case "byte": - field.setByte(model, Byte.valueOf(properties.getString(fillName))); - break; - case "short": - field.setShort(model, Short.valueOf(properties.getString(fillName))); - break; - case "int": - field.setInt(model, Integer.parseInt(properties.getString(fillName))) ; - break; - case "long": - field.setLong(model, properties.getLong(fillName)); - break; - case "double": - field.setDouble(model, Double.valueOf(properties.getString(fillName))); - break; - case "float": - field.setFloat(model, Float.valueOf(properties.getString(fillName))); - break; - case "boolean": - field.setBoolean(model, Boolean.parseBoolean(properties.getString(fillName))); - break; - case "class java.util.Date": - Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName)); - field.set(model,date) ; - break; - default: - String tn = field.getType().getTypeName(); - if("java.util.List".equals(tn)){ - String[] arr = fillName.split("\\["); - int index = 0; - String suffix; - String fastSuffix; - List list = new ArrayList<>(); - do{ - suffix = "["+index+"]"; - fastSuffix = "["+(index+1)+"]"; - list.add(properties.getString(arr[0]+suffix)); - index ++; - }while (properties.getString(arr[0]+fastSuffix) != null); - field.set(model, list); - }else if("java.util.Map".equals(tn)){ - String val = properties.getString(fillName); - field.set(model,FastJsonUtils.toMap(val)); - }else if(field.getType().isArray()){ - String val = properties.getString(fillName); - field.set(model,FastJsonUtils.toArray(val)); - }else{ - String[] ar = type.split(" "); - Object tinyObj = Class.forName(ar[1]).newInstance(); - invoke(tinyObj, properties, prefix); - field.set(model,tinyObj); - } - } - } - - } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java index 4e27fb8..48ef52f 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java @@ -1,12 +1,15 @@ package com.jd.platform.jlog.common.utils; import com.jd.platform.jlog.common.constant.Constant; +import com.jd.platform.jlog.common.handler.JcProperties; import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ThreadLocalRandom; @@ -146,4 +149,88 @@ public class ConfigUtil { chars[0] += 32; return String.valueOf(chars); } + + + /** + * 只支持简单的对象形配置 + * @param model bean + * @param properties 配置 + * @param prefix 前缀 + */ + @Deprecated + public static void invoke(Object model, JcProperties properties, String prefix) throws + IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException { + + Class clz = model.getClass(); + Field[] fields = model.getClass().getDeclaredFields(); + for (Field field : fields) { + String type = field.getGenericType().toString(); + field.setAccessible(true); + + String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName())); + + prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix; + String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName(); + + switch (type){ + case "class java.lang.String": + field.set(model, properties.getString(fillName)) ; + break; + case "byte": + field.setByte(model, Byte.valueOf(properties.getString(fillName))); + break; + case "short": + field.setShort(model, Short.valueOf(properties.getString(fillName))); + break; + case "int": + field.setInt(model, Integer.parseInt(properties.getString(fillName))) ; + break; + case "long": + field.setLong(model, properties.getLong(fillName)); + break; + case "double": + field.setDouble(model, Double.valueOf(properties.getString(fillName))); + break; + case "float": + field.setFloat(model, Float.valueOf(properties.getString(fillName))); + break; + case "boolean": + field.setBoolean(model, Boolean.parseBoolean(properties.getString(fillName))); + break; + case "class java.util.Date": + Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName)); + field.set(model,date) ; + break; + default: + String tn = field.getType().getTypeName(); + if("java.util.List".equals(tn)){ + String[] arr = fillName.split("\\["); + int index = 0; + String suffix; + String fastSuffix; + List list = new ArrayList<>(); + do{ + suffix = "["+index+"]"; + fastSuffix = "["+(index+1)+"]"; + list.add(properties.getString(arr[0]+suffix)); + index ++; + }while (properties.getString(arr[0]+fastSuffix) != null); + field.set(model, list); + }else if("java.util.Map".equals(tn)){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toMap(val)); + }else if(field.getType().isArray()){ + String val = properties.getString(fillName); + field.set(model,FastJsonUtils.toArray(val)); + }else{ + String[] ar = type.split(" "); + Object tinyObj = Class.forName(ar[1]).newInstance(); + invoke(tinyObj, properties, prefix); + field.set(model,tinyObj); + } + } + } + + } + } diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java index d671394..573999b 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java @@ -1,12 +1,22 @@ package com.jd.platform.jlog.config.apollo; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.concurrent.*; +import com.alibaba.fastjson.JSON; +import com.ctrip.framework.apollo.ConfigFile; +import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; import com.ctrip.framework.apollo.core.utils.StringUtils; import com.ctrip.framework.apollo.enums.PropertyChangeType; import com.ctrip.framework.apollo.model.ConfigChange; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.handler.TagConfig; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.core.*; import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.ConfigService; @@ -15,7 +25,9 @@ import org.slf4j.LoggerFactory; import static com.ctrip.framework.apollo.core.ApolloClientSystemConsts.APP_ID; +import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke; import static com.jd.platform.jlog.config.apollo.ApolloConstant.*; +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; /** @@ -32,10 +44,14 @@ public class ApolloConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; + + /** * 里面有resourceProperties 和 configProperties */ private static volatile Config config; + + private static volatile ApolloConfigurator instance; private ApolloConfigurator() { @@ -45,19 +61,19 @@ public class ApolloConfigurator implements Configurator { if (config == null) { // apollo的监听是按照namespace维度 config = ConfigService.getConfig(DEFAULT_NAMESPACE); + ApolloListener apolloListener = new ApolloListener(); config.addChangeListener(changeEvent -> { LOGGER.info("Apollo收到事件变更, keys={}", changeEvent.changedKeys()); for (String key : changeEvent.changedKeys()) { ConfigChange change = changeEvent.getChange(key); - ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), - change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType())); - new ApolloListener().onProcessEvent(event); + ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType())); + apolloListener.onProcessEvent(event); } }); } } } - System.out.println("Apollo配置器构建完成"); + LOGGER.info("Apollo配置器构建完成"); } @@ -74,22 +90,22 @@ public class ApolloConfigurator implements Configurator { @Override public String getString(String key) { - return null; + return config.getProperty(key, null); } @Override public Long getLong(String key) { - return null; + return config.getLongProperty(key,null); } @Override public List getList(String key) { - return null; + return FastJsonUtils.toList(config.getProperty(key,""), String.class) ; } @Override public T getObject(String key, Class clz) { - return null; + return FastJsonUtils.toBean(config.getProperty(key,""), clz); } @Override @@ -97,7 +113,6 @@ public class ApolloConfigurator implements Configurator { return false; } - @Override public boolean putConfig(String dataId, String content, long timeoutMills) { return false; @@ -146,4 +161,5 @@ public class ApolloConfigurator implements Configurator { System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getString(APOLLO_CONFIG_SERVICE)); } } + } diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java index 9861134..03c0faf 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java @@ -17,5 +17,4 @@ public class ApolloConstant { static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.config-service"; static final String PROP_APOLLO_SECRET = "apollo.accesskey.secret"; static final String PROP_APOLLO_CLUSTER = "apollo.cluster"; - static final String DEFAULT_NAMESPACE = "jLog"; } diff --git a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java index cbcccdc..a550b3d 100644 --- a/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java +++ b/config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java @@ -1,5 +1,6 @@ package com.jd.platform.jlog.config.apollo; +import com.jd.platform.jlog.core.ClientHandlerBuilder; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; @@ -13,6 +14,6 @@ public class ApolloListener implements ConfigChangeListener { @Override public void onChangeEvent(ConfigChangeEvent event) { - LOGGER.info("APOLLO 重写的事件 event={}", event.toString()); + ClientHandlerBuilder.refresh(); } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java index a087697..9621b03 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java @@ -14,18 +14,50 @@ public class ClientHandler { public static Map processReq(Map reqMap){ Map map = ExtractHandler.extractReqTag(reqMap); - return CompressHandler.compressReq(map);; + return CompressHandler.compressReq(map); } - public static Map processLog(){ - - return null; + public static Outcome processLog(String content){ + Map tagMap = ExtractHandler.extractLogTag(content); + return new Outcome(tagMap, null); } - public static Map processResp(byte[] resp, String content){ - byte[] contentBytes - return 1; + public static Outcome processResp(byte[] resp, Map respMap){ + Map tagMap = ExtractHandler.extractRespTag(respMap); + byte[] newContent = CompressHandler.compressResp(resp); + return new Outcome(tagMap, newContent); } + + + + + + public static class Outcome{ + + Map map; + byte[] content; + + public Outcome(Map map, byte[] content) { + this.map = map; + this.content = content; + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + public byte[] getContent() { + return content; + } + + public void setContent(byte[] content) { + this.content = content; + } + } } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java index c98663d..7af5130 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java @@ -4,22 +4,24 @@ import com.jd.platform.jlog.common.handler.CompressHandler; import com.jd.platform.jlog.common.handler.ExtractHandler; import com.jd.platform.jlog.common.handler.TagConfig; +import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD; +import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_LOG_RESP; + /** * @author tangbohu * @version 1.0.0 - * @ClassName TagHandlerBuilder.java + * @ClassName ClientHandlerBuilder.java * @Description TODO * @createTime 2022年03月05日 22:07:00 */ public class ClientHandlerBuilder { - public static void buildHandler(TagConfig tagConfig, Configurator configurator){ if(tagConfig == null){ tagConfig = buildTagConfigByConfigurator(configurator); } - ExtractHandler.buildTagHandler(tagConfig); + ExtractHandler.buildExtractHandler(tagConfig); CompressHandler.buildCompressHandler(configurator.getLong("compress"), configurator.getLong("threshold")); } @@ -30,6 +32,7 @@ public class ClientHandlerBuilder { throw new RuntimeException("configurator is null"); } ExtractHandler.refresh(buildTagConfigByConfigurator(configurator)); + CompressHandler.refresh(configurator.getLong("compress"), configurator.getLong("threshold") ); } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java index 7953690..de81b05 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java @@ -1,6 +1,8 @@ package com.jd.platform.jlog.core; +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; + /** * @author tangbohu * @version 1.0.0 @@ -14,7 +16,6 @@ public class ConfigChangeEvent { private String newValue; private String namespace; private ConfigChangeType changeType; - private static final String DEFAULT_NAMESPACE = "jLog"; public ConfigChangeEvent(){ diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java index c87087f..5be575d 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java @@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicBoolean; /** @@ -26,6 +27,9 @@ public class ConfiguratorFactory { public static volatile Configurator base = null; + public static AtomicBoolean useFileConfig = new AtomicBoolean(true); + + public static Configurator getInstance(){ if (instance == null) { @@ -64,24 +68,10 @@ public class ConfiguratorFactory { //noinspection LoopStatementThatDoesntLoop for (ConfiguratorProvider provider : builders) { LOGGER.info("配置中心的配置器获取成功, 类型为:{}", provider.build().getType()); + useFileConfig.set(false); return provider.build(); } return base; } - - - private static Configurator refreshBaseConfiguration() { - - synchronized (Configurator.class){ - try { - base = new FileConfigurator(); - } catch (IOException e) { - LOGGER.info("文件配置器构建失败", e); - throw new RuntimeException("build file buildConfiguration fail", e); - } - } - return base; - } - } diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java index fb05839..e5f78e4 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java @@ -15,7 +15,7 @@ public class Constant { /** * 监听重读配置文件间隔 单位ms */ - static final long LISTENER_CONFIG_INTERVAL = 3000; + static final long LISTENER_CONFIG_INTERVAL = 10000; static final String CONFIG_FILE_PROPERTIES = "/application.properties"; @@ -26,16 +26,19 @@ public class Constant { static final String ENV = "env"; - static final long AWAIT_TIME = 3 * 1000; - static final String YML = "yml"; public static final long DEFAULT_TIMEOUT = 2 * 1000; + static final long AWAIT_TIME = 3 * 1000; + public static final String SERVER_ADDR_KEY = "serverAddr"; public static final String NAMESPACE_KEY = "namespace"; + public static final String DEFAULT_NAMESPACE = "jLog"; + + /** * 配置文件集合 */ diff --git a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java index 8d97407..325fb7c 100644 --- a/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java +++ b/config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java @@ -1,8 +1,8 @@ package com.jd.platform.jlog.core; import com.alibaba.fastjson.JSON; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.utils.CollectionUtil; -import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.common.utils.StringUtil; import io.netty.util.concurrent.DefaultThreadFactory; import org.slf4j.Logger; @@ -16,6 +16,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import static com.jd.platform.jlog.core.ConfigChangeListener.EXECUTOR_SERVICE; +import static com.jd.platform.jlog.core.ConfiguratorFactory.useFileConfig; import static com.jd.platform.jlog.core.Constant.*; @@ -61,6 +62,16 @@ public class FileConfigurator implements Configurator { } } LOGGER.info("合并后的配置:{}",PROPERTIES.toString()); + for (String file : CONFIG_FILES) { + file = StringUtil.isEmpty(env) ? file : file + "_" + env; + LISTENED_FILES.add(file); + if(FILELISTENER == null){ + synchronized (FileConfigurator.class){ + FILELISTENER = new FileListener(); + FILELISTENER.addListener(); + } + } + } } @@ -94,80 +105,21 @@ public class FileConfigurator implements Configurator { } - @Override - public boolean removeConfig(String key, long timeoutMills) { - return false; - } - - - @Override - public boolean removeConfig(String key) { - return false; - } - - - @Override - public void addConfigListener(String node) { - if (StringUtil.isBlank(node)) { - return; - } - String env = System.getenv(ENV); - for (String file : CONFIG_FILES) { - file = StringUtil.isEmpty(env) ? file : file + "_" + env; - if(node.equals(file)){ - LISTENED_FILES.add(node); - if(FILELISTENER == null){ - synchronized (FileConfigurator.class){ - FILELISTENER = new FileListener(); - FILELISTENER.addListener(); - } - } - } - } - } - - - - - @Override - public void removeConfigListener(String node) { - - LISTENED_FILES.remove(node); - if(LISTENED_FILES.isEmpty() && FILELISTENER != null){ - LOGGER.info("没有要监听的key了 关闭线程池"); - FILELISTENER.onShutDown(); - try { - if(!EXECUTOR_SERVICE.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ - // 超时的时候向线程池中所有的线程发出中断(interrupted)。 - EXECUTOR_SERVICE.shutdownNow(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - FILELISTENER = null; - } - } - - - - @Override - public List getConfigByPrefix(String prefix) { - return PROPERTIES.getStrList(prefix); - } - - @Override public String getType() { return "file"; } - class FileListener implements ConfigChangeListener { - FileListener() { - } + private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("fileListener", 1)); + + + FileListener() {} synchronized void addListener() { FILELISTENER.onProcessEvent(new ConfigChangeEvent()); @@ -178,11 +130,26 @@ public class FileConfigurator implements Configurator { public void onChangeEvent(ConfigChangeEvent event) { for(; !getExecutorService().isShutdown() && !getExecutorService().isShutdown(); checkAndConfigure()) { try { + if(!useFileConfig.get()){ + LOGGER.info("装配了配置中心, 文件配置器关闭"); + executor.shutdown(); + if(!executor.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){ + // 超时的时候向线程池中所有的线程发出中断(interrupted)。 + executor.shutdownNow(); + return; + } + } Thread.sleep(LISTENER_CONFIG_INTERVAL); } catch (InterruptedException ignored) { } } } + + @Override + public ExecutorService getExecutorService() { + return executor; + } + } @@ -211,11 +178,11 @@ public class FileConfigurator implements Configurator { } }); if(change.get()){ - // LOGGER.info("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); + LOGGER.debug("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES)); PROPERTIES.clear(); FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props)); LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES)); - // TagHandlerBuilder.refresh(); + ClientHandlerBuilder.refresh(); } } diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java index 914c749..85e77b2 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java @@ -12,6 +12,7 @@ import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.EtcdClient; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.utils.CollectionUtil; import com.jd.platform.jlog.common.utils.StringUtil; import com.jd.platform.jlog.core.*; @@ -40,11 +41,11 @@ public class EtcdConfigurator implements Configurator { private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; - public static final String ROOT = "/jLog"; + static final String ROOT = "/jLog"; - private static final String PROPERTIES_PATH = "/jLog/jLog.properties"; + static final String PROPERTIES_PATH = "/jLog/jLog.properties"; - private static Properties PROPERTIES = new Properties(); + static JcProperties PROPERTIES = new JcProperties(); private EtcdConfigurator() { LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getString(SERVER_ADDR_KEY)); @@ -62,7 +63,7 @@ public class EtcdConfigurator implements Configurator { e.printStackTrace(); } } - + LOGGER.info("初始化etcd配置", JSON.toJSONString(PROPERTIES)); new EtcdListener().onProcessEvent(new ConfigChangeEvent()); } @@ -78,25 +79,24 @@ public class EtcdConfigurator implements Configurator { return instance; } - @Override public String getString(String key) { - return null; + return PROPERTIES.getString(key); } @Override public Long getLong(String key) { - return null; + return PROPERTIES.getLong(key); } @Override public List getList(String key) { - return null; + return PROPERTIES.getStrList(key); } @Override public T getObject(String key, Class clz) { - return null; + return PROPERTIES.getBean(key, clz); } @Override diff --git a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java index c10b1b2..b6c172b 100644 --- a/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java +++ b/config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java @@ -3,19 +3,30 @@ package com.jd.platform.jlog.etcd; import com.google.protobuf.ByteString; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.EtcdClient; import com.ibm.etcd.client.kv.KvClient; import com.ibm.etcd.client.kv.WatchUpdate; +import com.jd.platform.jlog.common.handler.JcProperties; +import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.utils.StringUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.ConfigChangeType; import io.netty.util.concurrent.DefaultThreadFactory; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES; +import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES_PATH; import static com.jd.platform.jlog.etcd.EtcdConfigurator.ROOT; /** @@ -27,9 +38,6 @@ import static com.jd.platform.jlog.etcd.EtcdConfigurator.ROOT; */ public class EtcdListener implements ConfigChangeListener { private KvClient.WatchIterator iterator; - private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, - TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), - new DefaultThreadFactory("etcdListener", 1)); public EtcdListener() { @@ -63,13 +71,32 @@ public class EtcdListener implements ConfigChangeListener { @Override public void onChangeEvent(ConfigChangeEvent event) { - System.out.println("onChangeEvent 一次又一次的进入 ==> "+event.getKey()); - } - @Override - public ExecutorService getExecutorService() { - return executor; - } + RangeResponse rangeResponse = EtcdConfigurator.client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync(); + List keyValues = rangeResponse.getKvsList(); + if (CollectionUtil.isEmpty(keyValues)) { + return; + } + String val = keyValues.get(0).getValue().toStringUtf8(); + JcProperties props = new JcProperties(); + if(StringUtil.isNotBlank(val)){ + try { + props.load(new StringReader(val)); + } catch (IOException e) { + e.printStackTrace(); + } + Set diffKeys = CollectionUtil.diffKeys(props, PROPERTIES); + if(!diffKeys.isEmpty()){ + PROPERTIES = props; + for (String diffKey : diffKeys) { + LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", event.getKey(), diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey))); + } + ClientHandlerBuilder.refresh(); + } + } + } } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java index dad6364..b74fdab 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java @@ -1,7 +1,6 @@ package com.jd.platform.jlog.nacos; import java.io.*; -import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -12,8 +11,8 @@ import com.alibaba.nacos.api.config.ConfigService; import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.utils.StringUtil; -import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.slf4j.Logger; @@ -21,7 +20,6 @@ import org.slf4j.LoggerFactory; import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr; import static com.jd.platform.jlog.core.Constant.*; -import static com.jd.platform.jlog.nacos.NacosConstant.*; @@ -32,20 +30,20 @@ public class NacosConfigurator implements Configurator { private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class); - - private static volatile NacosConfigurator instance; - - private static final Configurator FILE_CONFIG = ConfiguratorFactory.base; private static volatile ConfigService configService; - static final ConcurrentMap KEY_DATAID_MAP = new ConcurrentHashMap<>(8); + static volatile JcProperties PROPERTIES = new JcProperties(); + + static final String JLOG_GROUP = "JLOG_GROUP"; - static volatile Properties PROPERTIES = new Properties(); + static final String DEFAULT_DATA_ID = "jLog.properties"; static NacosListener NACOSLISTENER = new NacosListener(); + private static volatile NacosConfigurator instance; + public static NacosConfigurator getInstance() { if (instance == null) { @@ -69,13 +67,7 @@ public class NacosConfigurator implements Configurator { PROPERTIES.load(new StringReader(config)); LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(PROPERTIES)); configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, NACOSLISTENER); - Enumeration e = PROPERTIES.propertyNames(); - while (e.hasMoreElements()) { - KEY_DATAID_MAP.put((String) e.nextElement(), DEFAULT_DATA_ID); - } } - configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, new NacosListener()); - } catch (NacosException | IOException e) { throw new RuntimeException(e); } @@ -85,22 +77,22 @@ public class NacosConfigurator implements Configurator { @Override public String getString(String key) { - return null; + return PROPERTIES.getString(key); } @Override public Long getLong(String key) { - return null; + return PROPERTIES.getLong(key); } @Override public List getList(String key) { - return null; + return PROPERTIES.getStrList(key); } @Override public T getObject(String key, Class clz) { - return null; + return PROPERTIES.getBean(key, clz); } @@ -113,16 +105,15 @@ public class NacosConfigurator implements Configurator { @Override public boolean putConfig(String key, String content, long timeoutMills) { boolean result = false; - String dataId = KEY_DATAID_MAP.get(key); - if(StringUtil.isEmpty(dataId)){ + if(StringUtil.isEmpty(key)){ return false; } try { if (!PROPERTIES.isEmpty()) { PROPERTIES.setProperty(key, content); - result = configService.publishConfig(dataId, JLOG_GROUP, formatConfigStr(PROPERTIES)); + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, formatConfigStr(PROPERTIES)); } else { - result = configService.publishConfig(dataId, JLOG_GROUP, content); + result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, content); } } catch (NacosException ex) { LOGGER.error(ex.getErrMsg()); @@ -139,14 +130,7 @@ public class NacosConfigurator implements Configurator { properties.setProperty(SERVER_ADDR_KEY, address); } - String namespace = FILE_CONFIG.getString(NAMESPACE_KEY); - if (namespace != null) { - properties.setProperty(NAMESPACE_KEY, namespace); - }else{ - if (System.getProperty(NAMESPACE_KEY) != null) { - properties.setProperty(NAMESPACE_KEY, System.getProperty(NAMESPACE_KEY)); - } - } + properties.setProperty(NAMESPACE_KEY, DEFAULT_NAMESPACE); return properties; } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java deleted file mode 100644 index 513671f..0000000 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConstant.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.jd.platform.jlog.nacos; - - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName NacosConstant.java - * @Description TODO - * @createTime 2022年02月22日 10:22:00 - */ -public class NacosConstant { - - static final String JLOG_GROUP = "JLOG_GROUP"; - static final String DEFAULT_DATA_ID = "jLog.properties"; -} diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java index 06ed9f4..e1e4cc6 100644 --- a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java +++ b/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java @@ -1,10 +1,15 @@ package com.jd.platform.jlog.nacos; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; import com.alibaba.nacos.api.config.listener.AbstractSharedListener; +import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.api.naming.listener.Event; import com.alibaba.nacos.api.naming.listener.EventListener; import com.alibaba.nacos.common.utils.StringUtils; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import org.slf4j.Logger; @@ -15,8 +20,8 @@ import java.util.Properties; import java.util.Set; +import static com.jd.platform.jlog.core.Constant.*; import static com.jd.platform.jlog.nacos.NacosConfigurator.*; -import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID; /** @@ -32,12 +37,7 @@ public class NacosListener extends AbstractSharedListener implements ConfigChang @Override public void innerReceive(String dataId, String group, String configInfo) { - - LOGGER.info("configInfo:{}", configInfo); - if(!DEFAULT_DATA_ID.equals(dataId)){ - return; - } - + LOGGER.debug("configInfo:{}", configInfo); ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo).setNamespace(group); this.onProcessEvent(event); } @@ -46,10 +46,8 @@ public class NacosListener extends AbstractSharedListener implements ConfigChang @Override public void onChangeEvent(ConfigChangeEvent event) { - System.out.println("通用[配置]变更事件"); - LOGGER.info("通用[配置]变更事件 event:{}",event); - - Properties props = new Properties(); + LOGGER.debug("通用[配置]变更事件 event:{}",event); + JcProperties props = new JcProperties(); if (StringUtils.isNotBlank(event.getNewValue())) { try { props.load(new StringReader(event.getNewValue())); @@ -61,15 +59,18 @@ public class NacosListener extends AbstractSharedListener implements ConfigChang if(!diffKeys.isEmpty()){ PROPERTIES = props; for (String diffKey : diffKeys) { - LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", DEFAULT_DATA_ID, diffKey, new ConfigChangeEvent(diffKey, props.get(diffKey).toString(), PROPERTIES.get(diffKey).toString())); + LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", DEFAULT_DATA_ID, diffKey, + String.valueOf(props.get(diffKey)), + String.valueOf(PROPERTIES.get(diffKey))); } + ClientHandlerBuilder.refresh(); } } @Override public void onEvent(Event event) { - System.out.println("通用[服务]事件"); LOGGER.info("通用[服务]事件 event:{}",event); } + } diff --git a/config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java b/config/config-nacos/src/test/NacosTest.java similarity index 100% rename from config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosTest.java rename to config/config-nacos/src/test/NacosTest.java diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java index a6bd0db..ebaa734 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java @@ -2,23 +2,20 @@ package com.jd.platform.jlog.zk; import java.io.ByteArrayInputStream; import java.util.List; -import java.util.Properties; - import com.alibaba.fastjson.JSON; -import com.jd.platform.jlog.common.utils.FastJsonUtils; +import com.jd.platform.jlog.common.handler.JcProperties; import com.jd.platform.jlog.common.utils.StringUtil; import com.jd.platform.jlog.core.*; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte; import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT; +import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE; import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY; -import static com.jd.platform.jlog.zk.ZkConstant.*; /** @@ -32,10 +29,9 @@ public class ZkConfigurator implements Configurator { static volatile CuratorFramework zkClient; - static volatile Properties PROPERTIES = new Properties(); - - private static volatile ZkListener ZKLISTENER = null; + static volatile JcProperties PROPERTIES = new JcProperties(); + private static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; public ZkConfigurator() throws Exception { @@ -46,16 +42,13 @@ public class ZkConfigurator implements Configurator { .sessionTimeoutMs(6000) // 会话超时时间 .connectionTimeoutMs(2000) - .namespace(NAMESPACE) + .namespace(DEFAULT_NAMESPACE) // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 .retryPolicy(new ExponentialBackoffRetry(1000, 3)) .build(); zkClient.start(); } - if(zkClient.checkExists().forPath(DEFAULT_WORKER_PATH) == null){ - zkClient.create().withMode(CreateMode.EPHEMERAL).forPath(DEFAULT_WORKER_PATH); - } loadZkData(); new ZkListener(DEFAULT_CONFIG_PATH); LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(PROPERTIES)); @@ -65,22 +58,22 @@ public class ZkConfigurator implements Configurator { @Override public String getString(String key) { - return getString(key); + return PROPERTIES.getString(key); } @Override public Long getLong(String key) { - return null; + return PROPERTIES.getLong(key); } @Override public List getList(String key) { - return null; + return PROPERTIES.getStrList(key); } @Override public T getObject(String key, Class clz) { - return null; + return PROPERTIES.getBean(key, clz); } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java index 4f959e0..e2355e5 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java @@ -5,6 +5,9 @@ import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorProvider; +/** + * @author tangbohu + */ public class ZkConfiguratorProvider implements ConfiguratorProvider { @Override public Configurator build() { diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java deleted file mode 100644 index c4ac939..0000000 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConstant.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.jd.platform.jlog.zk; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName ZkConstant.java - * @Description TODO - * @createTime 2022年02月22日 10:16:00 - */ -public class ZkConstant { - - static final String NAMESPACE = "jLog"; - static final String DEFAULT_CONFIG_PATH = "/jLog.properties"; - static final String DEFAULT_WORKER_PATH = "/workers"; -} diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java index 9877341..e41b076 100644 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java +++ b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.zk; import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.core.ClientHandlerBuilder; import com.jd.platform.jlog.core.ConfigChangeEvent; import com.jd.platform.jlog.core.ConfigChangeListener; import com.jd.platform.jlog.core.ConfigChangeType; @@ -34,40 +35,23 @@ public class ZkListener implements ConfigChangeListener { cache = new NodeCache(zkClient, path); try { cache.start(true); + cache.getListenable().addListener(() -> onChangeEvent(null)); } catch (Exception e) { e.printStackTrace(); } - cache.getListenable().addListener(() -> { - String value = null; - if(null!=cache.getCurrentData()){ - value = new String(cache.getCurrentData().getData()); - } - onChangeEvent(null); - System.out.println("=####====== "+value); - }); } - @Override - public void onShutDown(){ - LOGGER.info("ZK删除监听器"); - try { - cache.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - @Override public void onChangeEvent(ConfigChangeEvent event) { LOGGER.info("ZK数据变更-当前监听器关注的path:{} PROPERTIES:{}", path, JSON.toJSONString(PROPERTIES)); Properties props = new Properties(); props.putAll(PROPERTIES); try { - LOGGER.info("ZK数据变更,旧Properties:{}", JSON.toJSONString(props)); + LOGGER.debug("ZK数据变更,旧Properties:{}", JSON.toJSONString(props)); loadZkData(); - LOGGER.info("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES)); + LOGGER.debug("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES)); } catch (Exception e) { e.printStackTrace(); } @@ -77,8 +61,9 @@ public class ZkListener implements ConfigChangeListener { LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey, new ConfigChangeEvent(diffKey, String.valueOf(props.get(diffKey)), - String.valueOf(PROPERTIES.get(diffKey).toString()))); + String.valueOf(PROPERTIES.get(diffKey)))); } + ClientHandlerBuilder.refresh(); } } diff --git a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java b/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java deleted file mode 100644 index f66b9ec..0000000 --- a/config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.jd.platform.jlog.zk; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.recipes.cache.NodeCache; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author tangbohu - * @version 1.0.0 - * @ClassName ZkTest.java - * @Description TODO - * @createTime 2022年02月22日 17:26:00 - */ -public class ZkTest { - private final static Logger log = LoggerFactory.getLogger(ZkTest.class); - - - public static void main(String[] args) throws Exception { - CuratorFramework client = CuratorFrameworkFactory.builder().connectString("101.42.242.201:2181") - // 连接超时时间 - .sessionTimeoutMs(2000) - // 会话超时时间 - .connectionTimeoutMs(6000) - // .namespace("jLog") - // 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次 - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build(); - //开启客户端 - client.start(); - //创建缓存节点 - NodeCache nodeCache = new NodeCache(client, "/jLog/jLog.properties"); - - //将该节点数据初始化到本地缓存 - nodeCache.start(true); - //添加节点监听事件,NodeCacheListener每次都会触发，但不能获取监听的操作类型到底是添加还是删除等。 - nodeCache.getListenable().addListener(() -> { - String value = null; - System.out.println("Listener-name: "+Thread.currentThread().getName()); - if(null!=nodeCache.getCurrentData()){ - value = new String(nodeCache.getCurrentData().getData()); - } - System.out.println("======= "+value); - }); - - - client.setData().forPath("/jLog/jLog.properties","noweeee".getBytes()); - //睡眠等待监听事件触发 - Thread.sleep(25000); - System.out.println("main-name: "+Thread.currentThread().getName()); - - //关闭 - nodeCache.close(); - client.close(); - - } - -} diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java index 70417b0..96c4de7 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java @@ -8,10 +8,9 @@ import com.jd.platform.jlog.client.log.LogExceptionStackTrace; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; import com.jd.platform.jlog.common.model.RunLogMessage; -import com.jd.platform.jlog.common.handler.TagHandler; +import com.jd.platform.jlog.core.ClientHandler; import org.slf4j.helpers.MessageFormatter; -import java.util.Map; /** * className：TracerLog4JAppender @@ -60,8 +59,8 @@ public class TracerLogbackAppender extends AppenderBase { String formattedMessage = getMessage(loggingEvent); logMessage.setContent(formattedMessage); - Map map = TagHandler.extractLogTag(formattedMessage); - logMessage.setTagMap(map); + ClientHandler.Outcome out = ClientHandler.processLog(formattedMessage); + logMessage.setTagMap(out.getMap()); return logMessage; } diff --git a/example/src/main/resources/application.properties b/example/src/main/resources/application.properties index 3f355de..19b36d2 100644 --- a/example/src/main/resources/application.properties +++ b/example/src/main/resources/application.properties @@ -1,19 +1,6 @@ server.port=8085 +serverAddr=101.42.242.201:2379 -tag-config.reqTags[0]=uid -tag-config.reqTags[1]=url - -tag-config.logTags[0]=node -tag-config.logTags[1]=bizType - -tag-config.respTags[0]=errno -tag-config.respTags[1]=msg -tag-config.delimiter=| -tag-config.join== -tag-config.extract=41 - -compress=68 -threshold=10000 - -workers[0]=1 -workers[1]=2 +apollo.meta=http://127.0.0.1:8080 +apollo.config-service=http://127.0.0.1:8080 +app.id=order \ No newline at end of file diff --git a/example/src/main/resources/application.ymlbak b/example/src/main/resources/application.ymlbak deleted file mode 100644 index e0a50af..0000000 --- a/example/src/main/resources/application.ymlbak +++ /dev/null @@ -1,28 +0,0 @@ -serverAddr: 101.42.242.201:2181 -jlog: - center-config: - center: etcd - etcdServer: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ - nacosServer: 101.42.242.201:8848 - zkServer: 101.42.242.201:2181 - tag-config: - reqTags: - - tag1 - - tag2 - logTags: - - tag3 - - tag4 - regex: "" - delimiter: | - join: = - - - -server: - port: 8081 - - server.port=8083 - - #apollo.meta=http://127.0.0.1:8080 - #apollo.config-service=http://127.0.0.1:8080 - #app.id=order \ No newline at end of file diff --git a/example/src/main/resources/bakapplication.yml b/example/src/main/resources/bakapplication.yml index bce0acf..2bb1bcb 100644 --- a/example/src/main/resources/bakapplication.yml +++ b/example/src/main/resources/bakapplication.yml @@ -1,23 +1,24 @@ -serverAddr: http://101.42.242.201:8848 +workers: [1,2,3] +compress: 68 +threshold: 10000 + +serverAddr: 101.42.242.201:2181 server: port: 8085 test: true -workers: - - 111 - - 222 - - 333 tag-config: reqTags: - uid - - ip + - url - newK logTags: - - errno - node - regex: "" + - bizType + respTags: + - errno delimiter: "|" - join: = + join: "=" diff --git a/example/src/main/resources/bakjLog.properties b/example/src/main/resources/bakjLog.properties index ef21cc4..697d627 100644 --- a/example/src/main/resources/bakjLog.properties +++ b/example/src/main/resources/bakjLog.properties @@ -1,9 +1,18 @@ server.port=8085 -tag-config.reqTags=["uid","url"] -tag-config.logTags=["node","bizType"] -tag-config.respTags=["errno","msg"] -tag-config.regex="" -tag-config.delimiter="|" -tag-config.join="=" + +tag-config.reqTags[0]=uid +tag-config.reqTags[1]=url + +tag-config.logTags[0]=node +tag-config.logTags[1]=bizType + +tag-config.respTags[0]=errno +tag-config.respTags[1]=msg +tag-config.delimiter=| +tag-config.join== tag-config.extract=41 -tag-config.compress=81 \ No newline at end of file + +compress=68 +threshold=10000 + +workers=[1,2,3] diff --git a/example/src/test/java/com/jd/platform/jlog/test/Common.java b/example/src/test/java/com/jd/platform/jlog/test/Common.java index 3f10c42..b6034fb 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/Common.java +++ b/example/src/test/java/com/jd/platform/jlog/test/Common.java @@ -27,7 +27,7 @@ public class Common { LOGGER.info("配置器类型：{}", configurator.getType()); String addr = configurator.getString("serverAddr"); LOGGER.info("配置器get addr：{}", addr); - TagConfig tagConfig = configurator.getObject("tagConfig", TagConfig.class); + TagConfig tagConfig = configurator.getObject("tag-config", TagConfig.class); LOGGER.info("配置器get tagConfig：{}", tagConfig.toString()); List workers = configurator.getList("workers"); LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers)); diff --git a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java index 52e7456..e3eca92 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java @@ -50,7 +50,7 @@ public class FileConfiguratorTest { @Test public void testAddConfigListener() throws Exception { - String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties"; + String path = "/Users/didi/Desktop/jlog/example/target/classes/bakapplication.yml"; Properties props = new Properties(); FileInputStream fis = new FileInputStream(new File(path)); if (path.contains("yml")) { @@ -61,20 +61,7 @@ public class FileConfiguratorTest { LOGGER.info("读取文件：{} 最新配置:{}", path, JSON.toJSONString(props)); modifyFile(path); LOGGER.info("修改文件完毕 准备触发监听器"); - Thread.sleep(2000); - LOGGER.info("睡醒了 应该更新了配置，testKey：{}",configurator.getString("testKey")); - Thread.sleep(2000); - LOGGER.info("移除监听器之前，testKey：{}",configurator.getString("testKey")); - LOGGER.info("移除监听器之后，testKey：{}",configurator.getString("testKey")); - modifyFile(path); - LOGGER.info("修改文件完毕 准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey")); - - - LOGGER.info("再次添加监听器"); - modifyFile(path); - LOGGER.info("修改文件完毕"); - Thread.sleep(1000); - + Thread.sleep(10000); } diff --git a/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java index bbd1788..26582a4 100644 --- a/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java +++ b/example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java @@ -48,9 +48,8 @@ public class ZKConfiguratorTest { public void testUpdateCFG() throws Exception { List workers = configurator.getList("workers"); LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers)); - String myIp = "121.1.1.0"; + String myIp = "121.0"; if(workers.contains(myIp)){ - // do nothing LOGGER.info("自己的IP还在配置list里 什么也不做"); return; }else{ diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java index d716012..877838d 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java @@ -1,12 +1,16 @@ package com.jd.platform.jlog.worker.config; +import com.alibaba.fastjson.JSON; import com.jd.platform.jlog.common.constant.Constant; import com.jd.platform.jlog.common.utils.IpUtils; +import com.jd.platform.jlog.core.Configurator; import com.jd.platform.jlog.core.ConfiguratorFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.Resource; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -42,12 +46,16 @@ public class CenterStarter { */ public void uploadSelfInfo() { //开启上传worker信息 + Configurator config = ConfiguratorFactory.getInstance(); ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); scheduledExecutorService.scheduleAtFixedRate(() -> { try { - ConfiguratorFactory.getInstance().putConfig(buildKey(), buildValue()); - ConfiguratorFactory.getInstance().putConfig(buildSecondKey(), buildValue()); + List list = config.getList("workers"); + if(!list.contains(buildKey())){ + list.add(buildValue()); + } + config.putConfig("workers", JSON.toJSONString(list)); } catch (Exception e) { //do nothing e.printStackTrace(); @@ -64,14 +72,6 @@ public class CenterStarter { return Constant.WORKER_PATH + workerPath + "/" + hostName; } - /** - * 在配置中心对应机房存放的key - */ - private String buildSecondKey() { - String hostName = IpUtils.getHostName(); - return Constant.WORKER_PATH + workerPath + "/" + mdc + "/" + hostName; - } - /** * 在配置中心存放的value */ diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java deleted file mode 100644 index abae934..0000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/ConfigCenter.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jd.platform.jlog.worker.config; - - -import com.jd.platform.jlog.common.handler.TagConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - - -/** - * EtcdConfig - * @author wuweifeng wrote on 2019-12-06 - * @version 1.0 - */ -@Component -@ConfigurationProperties(prefix = "jlog") -public class ConfigCenter { - - private Logger logger = LoggerFactory.getLogger(getClass()); - - private TagConfig tagConfig ; - - public TagConfig getTagConfig() { - return tagConfig; - } - - public void setTagConfig(TagConfig tagConfig) { - this.tagConfig = tagConfig; - } - -} diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java index 5ac5061..167bcc0 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java @@ -104,8 +104,8 @@ public class TracerConsumer implements WorkHandler { map.put("threadName", runLogMessage.getThreadName()); map.put("methodName", runLogMessage.getMethodName()); map.put("logLevel", runLogMessage.getLogLevel()); - // map.put("createTime", DateUtil.formatDateTime(new Date(runLogMessage.getCreateTime()))); map.put("content", runLogMessage.getContent()); + map.putAll(runLogMessage.getTagMap()); tracerLogToDbStore.offer(map); } @@ -115,9 +115,12 @@ public class TracerConsumer implements WorkHandler { * 处理filter里处理的出入参 */ private void dealFilterModel(TracerBean tracerBean) { + List> mapList = tracerBean.getTracerObject(); Map requestMap = mapList.get(0); + Map map = new HashMap<>(requestMap); + long tracerId = requestMap.get("tracerId") == null ? 0 : Long.valueOf(requestMap.get("tracerId").toString()); //filter的出入参 Map responseMap = mapList.get(mapList.size() - 1); @@ -127,59 +130,11 @@ public class TracerConsumer implements WorkHandler { responseBytes = (byte[]) responseMap.get("response"); } - Map map = new HashMap<>(); - //jsf的是用户自己设置的request入参，http的是从httpRequest读取的 - if (requestMap.get("wholeRequest") == null) { - map.put("requestContent", FastJsonUtils.collectToString(requestMap)); - } else { - map.put("requestContent", requestMap.get("wholeRequest")); - } - - //此处做了一个base64编码，否则原编码直接进去，取出来后是String，直接getBytes后无法用Zstd解压 map.put("responseContent", responseBytes); - // map.put("createTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); map.put("costTime", tracerBean.getCostTime()); - - map.put("tracerId", tracerId); - - String pin = requestMap.get("pin") == null ? "" : requestMap.get("pin").toString(); - map.put("pin", pin); - - String uri = requestMap.get("uri") == null ? "" : requestMap.get("uri").toString(); - map.put("uri", uri); - - //appName - String appName = requestMap.get("appName") == null ? "" : requestMap.get("appName").toString(); - map.put("appName", appName); - - String openudid = requestMap.get("openudid") == null ? "" : requestMap.get("openudid").toString(); - - if (StringUtil.isNullOrEmpty(openudid)) { - String uuid = requestMap.get("uuid") == null ? "" : requestMap.get("uuid").toString(); - map.put("uuid", uuid); - } else { - map.put("uuid", openudid); - } - - String client = requestMap.get("client") == null ? "" : requestMap.get("client").toString(); - int clientType = 0; - if ("apple".equals(client)) { - clientType = 2; - } else if ("android".equals(client)) { - clientType = 1; - } - map.put("clientType", clientType); - String clientVersion = requestMap.get("clientVersion") == null ? "" : requestMap.get("clientVersion").toString(); - map.put("clientVersion", clientVersion); - - String userIp = requestMap.get("ip") == null ? "" : requestMap.get("ip").toString(); - map.put("userIp", userIp); - String serverIp = requestMap.get("serverIp") == null ? "" : requestMap.get("serverIp").toString(); - map.put("serverIp", serverIp); - - // map.put("intoDbTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime()))); - + responseMap.remove("response"); + map.putAll(responseMap); tracerModelToDbStore.offer(map); } diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java b/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java deleted file mode 100644 index b359ade..0000000 --- a/worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.jd.platform.jlog.worker.entity; - - -import java.io.Serializable; - -/** - * 入库、检索时用的对象 - * - * @author wuweifeng - * @version 1.0 - * @date 2021-08-20 - */ -public class TracerModel implements Serializable { - /** - * tracerId - */ - private long tracerId; - /** - * 请求的入参 - */ - private String requestContent; - /** - * 响应的出参 - */ - private String responseContent; - /** - * 日志请求时间（数据库里存的是DateTime，2021-08-24 19:47:30.0） - */ - private long createTime; - /** - * 请求耗时（即响应时间戳减去请求时间戳） - */ - private int costTime; - /** - * 用户pin - */ - private String pin; - /** - * 用户uuid - */ - private String uuid; - /** - * Android=1、ios=2 - */ - private int clientType; - /** - * 客户端版本号，9.3.6 - */ - private String clientVersion; - /** - * 用户ip - */ - private String userIp; - /** - * 服务端ip - */ - private String serverIp; - /** - * 入库时间 - */ - private long intoDbTime; - - - public long getIntoDbTime() { - return intoDbTime; - } - - public void setIntoDbTime(long intoDbTime) { - this.intoDbTime = intoDbTime; - } - - public String getRequestContent() { - return requestContent; - } - - public void setRequestContent(String requestContent) { - this.requestContent = requestContent; - } - - public String getResponseContent() { - return responseContent; - } - - public void setResponseContent(String responseContent) { - this.responseContent = responseContent; - } - - public long getTracerId() { - return tracerId; - } - - public void setTracerId(long tracerId) { - this.tracerId = tracerId; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - public int getCostTime() { - return costTime; - } - - public void setCostTime(int costTime) { - this.costTime = costTime; - } - - public String getPin() { - return pin; - } - - public void setPin(String pin) { - this.pin = pin; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public int getClientType() { - return clientType; - } - - public void setClientType(int clientType) { - this.clientType = clientType; - } - - public String getClientVersion() { - return clientVersion; - } - - public void setClientVersion(String clientVersion) { - this.clientVersion = clientVersion; - } - - - public String getUserIp() { - return userIp; - } - - public void setUserIp(String userIp) { - this.userIp = userIp; - } - - public String getServerIp() { - return serverIp; - } - - public void setServerIp(String serverIp) { - this.serverIp = serverIp; - } -} -- Gitee From 166ee440df7f92a95a4f40300f84e399fcf2a42a Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Mon, 14 Mar 2022 20:45:58 +0800 Subject: [PATCH 21/22] =?UTF-8?q?=E5=85=A8=E9=9D=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- worker/src/main/resources/application.yml | 6 ++---- worker/src/main/resources/logback-spring.xml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index c0030ca..924c7ae 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -6,10 +6,8 @@ queue: maxSize: ${queueSize:16384} preDbSize: ${preDbSize:10000} #etcd的地址，如有多个用逗号分隔 -jConfig: - mdc: ${mdc:default} - server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！ - workerPath: ${workerPath:default} #该worker放到哪个path下，譬如放/app1下，则该worker只能被app1使用，不会为其他client提供服务 +serverAddr: 101.42.242.201:2181 + server: port: 8080 #ck信息，自行修改 diff --git a/worker/src/main/resources/logback-spring.xml b/worker/src/main/resources/logback-spring.xml index 557bfde..31d2e34 100644 --- a/worker/src/main/resources/logback-spring.xml +++ b/worker/src/main/resources/logback-spring.xml @@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息， --> - + -- Gitee From cbd7fc7c9828ca169585fcb6194576f44c974f9d Mon Sep 17 00:00:00 2001 From: liyunfeng Date: Mon, 14 Mar 2022 20:46:28 +0800 Subject: [PATCH 22/22] =?UTF-8?q?=E5=85=A8=E9=9D=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/jlog/worker/config/CenterStarter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java index 877838d..9c234f4 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java @@ -54,8 +54,8 @@ public class CenterStarter { List list = config.getList("workers"); if(!list.contains(buildKey())){ list.add(buildValue()); + config.putConfig("workers", JSON.toJSONString(list)); } - config.putConfig("workers", JSON.toJSONString(list)); } catch (Exception e) { //do nothing e.printStackTrace(); -- Gitee