From 25966c9301180bb1bc9d336c76be296712e90ebb Mon Sep 17 00:00:00 2001 From: luohaojie <1967544561@qq.com> Date: Tue, 18 Jan 2022 16:36:53 +0800 Subject: [PATCH 1/3] test --- .../src/main/java/com/java3y/austin/pending/Task.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/austin-handler/src/main/java/com/java3y/austin/pending/Task.java b/austin-handler/src/main/java/com/java3y/austin/pending/Task.java index 87941af..b0afa41 100644 --- a/austin-handler/src/main/java/com/java3y/austin/pending/Task.java +++ b/austin-handler/src/main/java/com/java3y/austin/pending/Task.java @@ -44,7 +44,7 @@ public class Task implements Runnable { return; } - // 1.平台通用去重 + // 1.平台通用去重 test deduplicationRuleService.duplication(taskInfo); // 2. 真正发送消息 From f2973035bb3557166fe8b3d855af697cf95f3944 Mon Sep 17 00:00:00 2001 From: luohaojie <1967544561@qq.com> Date: Tue, 18 Jan 2022 20:29:45 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E9=A2=91=E6=AC=A1=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../austin/domain/DeduplicationParam.java | 5 +- .../AbstractDeduplicationService.java | 2 +- .../deduplication/DeduplicationConstants.java | 15 +++++ .../DeduplicationRuleService.java | 65 ++++++++----------- .../deduplication/DeduplicationService.java | 12 ++++ .../service/deduplication/build/Builder.java | 12 ++++ .../deduplication/build/BuilderFactory.java | 40 ++++++++++++ .../build/ContentDeduplicationBuilder.java | 31 +++++++++ .../build/FrequencyDeduplicationBuilder.java | 34 ++++++++++ 9 files changed, 176 insertions(+), 40 deletions(-) create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java diff --git a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java index e976a32..1c65366 100644 --- a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java +++ b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java @@ -1,5 +1,6 @@ package com.java3y.austin.domain; +import com.alibaba.fastjson.annotation.JSONField; import com.java3y.austin.enums.AnchorState; import lombok.Builder; import lombok.Data; @@ -12,7 +13,6 @@ import lombok.Data; @Builder @Data public class DeduplicationParam { - /** * TaskIno信息 */ @@ -22,16 +22,17 @@ public class DeduplicationParam { * 去重时间 * 单位：秒 */ + @JSONField(name = "time") private Long deduplicationTime; /** * 需达到的次数去重 */ + @JSONField(name = "num") private Integer countNum; /** * 标识属于哪种去重 */ private AnchorState anchorState; - } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java index 912199a..d1fb8c6 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java @@ -19,7 +19,7 @@ import java.util.*; * 去重服务 */ @Slf4j -public abstract class AbstractDeduplicationService { +public abstract class AbstractDeduplicationService implements DeduplicationService { @Autowired private RedisUtils redisUtils; diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java new file mode 100644 index 0000000..b2cef57 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java @@ -0,0 +1,15 @@ +package com.java3y.austin.service.deduplication; + +/** + * @author huskey + * @date 2022/1/18 + */ +public class DeduplicationConstants { + + /** + * 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} + */ + public static final String DEDUPLICATION_RULE_KEY = "deduplication"; + public static final String CONTENT_DEDUPLICATION = "contentDeduplication"; + public static final String FREQUENCY_DEDUPLICATION = "frequencyDeduplication"; +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java index 52ca75b..2db2503 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java @@ -1,18 +1,17 @@ package com.java3y.austin.service.deduplication; -import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.spring.annotation.ApolloConfig; +import com.google.common.collect.Lists; import com.java3y.austin.constant.AustinConstant; import com.java3y.austin.domain.DeduplicationParam; import com.java3y.austin.domain.TaskInfo; -import com.java3y.austin.enums.AnchorState; +import com.java3y.austin.service.deduplication.build.BuilderFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; -import java.util.Date; +import java.util.List; /** * @author 3y. @@ -22,47 +21,39 @@ import java.util.Date; @Service public class DeduplicationRuleService { - /** - * 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - */ - private static final String DEDUPLICATION_RULE_KEY = "deduplication"; - private static final String CONTENT_DEDUPLICATION = "contentDeduplication"; - private static final String FREQUENCY_DEDUPLICATION = "frequencyDeduplication"; - private static final String TIME = "time"; - private static final String NUM = "num"; - @Autowired - private ContentDeduplicationService contentDeduplicationService; + private static final String SERVICE = "Service"; + @ApolloConfig("boss.austin") + private Config config; @Autowired - private FrequencyDeduplicationService frequencyDeduplicationService; + private ApplicationContext applicationContext; + @Autowired + private BuilderFactory builderFactory; - @ApolloConfig("boss.austin") - private Config config; + //需要去重的服务 + private static final List DEDUPLICATION_LIST = Lists.newArrayList(DeduplicationConstants.CONTENT_DEDUPLICATION, DeduplicationConstants.FREQUENCY_DEDUPLICATION); public void duplication(TaskInfo taskInfo) { - // 配置示例:{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - JSONObject property = JSON.parseObject(config.getProperty(DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT)); - JSONObject contentDeduplication = property.getJSONObject(CONTENT_DEDUPLICATION); - JSONObject frequencyDeduplication = property.getJSONObject(FREQUENCY_DEDUPLICATION); + // 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} + String deduplication = config.getProperty(DeduplicationConstants.DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT); + //去重 + DEDUPLICATION_LIST.forEach( + key -> { + DeduplicationParam deduplicationParam = builderFactory.select(key).build(deduplication, key); + if (deduplicationParam != null) { + deduplicationParam.setTaskInfo(taskInfo); + DeduplicationService deduplicationService = findService(key + SERVICE); + deduplicationService.deduplication(deduplicationParam); + } + } + ); - // 文案去重 - DeduplicationParam contentParams = DeduplicationParam.builder() - .deduplicationTime(contentDeduplication.getLong(TIME)) - .countNum(contentDeduplication.getInteger(NUM)).taskInfo(taskInfo) - .anchorState(AnchorState.CONTENT_DEDUPLICATION) - .build(); - contentDeduplicationService.deduplication(contentParams); + } + private DeduplicationService findService(String beanName) { + return applicationContext.getBean(beanName, DeduplicationService.class); - // 运营总规则去重(一天内用户收到最多同一个渠道的消息次数) - Long seconds = (DateUtil.endOfDay(new Date()).getTime() - DateUtil.current()) / 1000; - DeduplicationParam businessParams = DeduplicationParam.builder() - .deduplicationTime(seconds) - .countNum(frequencyDeduplication.getInteger(NUM)).taskInfo(taskInfo) - .anchorState(AnchorState.RULE_DEDUPLICATION) - .build(); - frequencyDeduplicationService.deduplication(businessParams); } } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java new file mode 100644 index 0000000..35c4b80 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java @@ -0,0 +1,12 @@ +package com.java3y.austin.service.deduplication; + +import com.java3y.austin.domain.DeduplicationParam; + +/** + * @author huskey + * @date 2022/1/18 + */ +public interface DeduplicationService { + + void deduplication(DeduplicationParam param); +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java new file mode 100644 index 0000000..cd78e38 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java @@ -0,0 +1,12 @@ +package com.java3y.austin.service.deduplication.build; + +import com.java3y.austin.domain.DeduplicationParam; + +/** + * @author luohaojie + * @date 2022/1/18 + */ +public interface Builder { + + DeduplicationParam build(String deduplication, String key); +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java new file mode 100644 index 0000000..795dafa --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java @@ -0,0 +1,40 @@ +package com.java3y.austin.service.deduplication.build; + +import com.java3y.austin.service.deduplication.DeduplicationConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + + +/** + * @author huskey + * @date 2022/1/18 + */ + +@Service +public class BuilderFactory { + + + private Map builderFactory = new HashMap<>(4); + + @Autowired + private Builder contentDeduplicationBuilder; + + @Autowired + private Builder frequencyDeduplicationBuilder; + + @PostConstruct + public void init() { + builderFactory.put(DeduplicationConstants.CONTENT_DEDUPLICATION, contentDeduplicationBuilder); + builderFactory.put(DeduplicationConstants.FREQUENCY_DEDUPLICATION, frequencyDeduplicationBuilder); + } + + + public Builder select(String key) { + return builderFactory.get(key); + } + +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java new file mode 100644 index 0000000..a542712 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java @@ -0,0 +1,31 @@ +package com.java3y.austin.service.deduplication.build; + +import com.alibaba.fastjson.JSONObject; +import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.enums.AnchorState; +import com.java3y.austin.service.deduplication.build.Builder; +import org.springframework.stereotype.Service; + + +/** + * @author huskey + * @date 2022/1/18 + */ + +@Service +public class ContentDeduplicationBuilder implements Builder { + + public DeduplicationParam build(String deduplication, String key) { + JSONObject object = JSONObject.parseObject(deduplication); + if (object == null) { + return null; + } + DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); + if (deduplicationParam == null) { + return null; + } + deduplicationParam.setAnchorState(AnchorState.RULE_DEDUPLICATION); + return deduplicationParam; + + } +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java new file mode 100644 index 0000000..972ed87 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java @@ -0,0 +1,34 @@ +package com.java3y.austin.service.deduplication.build; + +import cn.hutool.core.date.DateUtil; +import com.alibaba.fastjson.JSONObject; +import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.enums.AnchorState; +import com.java3y.austin.service.deduplication.build.Builder; +import org.springframework.stereotype.Service; + +import java.util.Date; + +/** + * @author huskey + * @date 2022/1/18 + */ + +@Service +public class FrequencyDeduplicationBuilder implements Builder { + + @Override + public DeduplicationParam build(String deduplication, String key) { + JSONObject object = JSONObject.parseObject(deduplication); + if (object == null) { + return null; + } + DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); + if (deduplicationParam == null) { + return null; + } + deduplicationParam.setDeduplicationTime((DateUtil.endOfDay(new Date()).getTime() - DateUtil.current()) / 1000); + deduplicationParam.setAnchorState(AnchorState.RULE_DEDUPLICATION); + return deduplicationParam; + } +} From c1ff4f3a94c5e5363d428bf9046276fbd52394eb Mon Sep 17 00:00:00 2001 From: 3y Date: Wed, 19 Jan 2022 22:55:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../austin/enums/DeduplicationType.java | 37 ++++++++++++++++ .../austin/domain/DeduplicationParam.java | 6 ++- .../deduplication/DeduplicationConstants.java | 15 ------- .../deduplication/DeduplicationHolder.java | 36 ++++++++++++++++ .../DeduplicationRuleService.java | 42 +++++++------------ .../build/AbstractDeduplicationBuilder.java | 40 ++++++++++++++++++ .../service/deduplication/build/Builder.java | 12 +++++- .../deduplication/build/BuilderFactory.java | 40 ------------------ .../build/ContentDeduplicationBuilder.java | 26 ++++++------ .../build/FrequencyDeduplicationBuilder.java | 17 ++++---- .../AbstractDeduplicationService.java | 15 ++++++- .../ContentDeduplicationService.java | 7 +++- .../{ => service}/DeduplicationService.java | 6 ++- .../FrequencyDeduplicationService.java | 7 +++- 14 files changed, 194 insertions(+), 112 deletions(-) create mode 100644 austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java delete mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java create mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java delete mode 100644 austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java rename austin-handler/src/main/java/com/java3y/austin/service/deduplication/{ => service}/AbstractDeduplicationService.java (89%) rename austin-handler/src/main/java/com/java3y/austin/service/deduplication/{ => service}/ContentDeduplicationService.java (79%) rename austin-handler/src/main/java/com/java3y/austin/service/deduplication/{ => service}/DeduplicationService.java (64%) rename austin-handler/src/main/java/com/java3y/austin/service/deduplication/{ => service}/FrequencyDeduplicationService.java (80%) diff --git a/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java b/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java new file mode 100644 index 0000000..3cf4b1c --- /dev/null +++ b/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java @@ -0,0 +1,37 @@ +package com.java3y.austin.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.List; + +/** + * 去重类型枚举 + * @author 3y + */ +@Getter +@ToString +@AllArgsConstructor +public enum DeduplicationType { + + CONTENT(10, "N分钟相同内容去重"), + FREQUENCY(20, "一天内N次相同渠道去重"), + ; + private Integer code; + private String description; + + + /** + * 获取去重渠道的列表 + * @return + */ + public static List getDeduplicationList() { + ArrayList result = new ArrayList<>(); + for (DeduplicationType value : DeduplicationType.values()) { + result.add(value.getCode()); + } + return result; + } +} diff --git a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java index 1c65366..8f73ea5 100644 --- a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java +++ b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java @@ -2,8 +2,10 @@ package com.java3y.austin.domain; import com.alibaba.fastjson.annotation.JSONField; import com.java3y.austin.enums.AnchorState; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; /** * @author 3y @@ -12,6 +14,8 @@ import lombok.Data; */ @Builder @Data +@AllArgsConstructor +@NoArgsConstructor public class DeduplicationParam { /** * TaskIno信息 @@ -32,7 +36,7 @@ public class DeduplicationParam { private Integer countNum; /** - * 标识属于哪种去重 + * 标识属于哪种去重(数据埋点) */ private AnchorState anchorState; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java deleted file mode 100644 index b2cef57..0000000 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.java3y.austin.service.deduplication; - -/** - * @author huskey - * @date 2022/1/18 - */ -public class DeduplicationConstants { - - /** - * 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - */ - public static final String DEDUPLICATION_RULE_KEY = "deduplication"; - public static final String CONTENT_DEDUPLICATION = "contentDeduplication"; - public static final String FREQUENCY_DEDUPLICATION = "frequencyDeduplication"; -} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java new file mode 100644 index 0000000..342c5a8 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java @@ -0,0 +1,36 @@ +package com.java3y.austin.service.deduplication; + +import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.service.deduplication.service.DeduplicationService; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + + +/** + * @author huskey + * @date 2022/1/18 + */ +@Service +public class DeduplicationHolder { + + private Map builderHolder = new HashMap<>(4); + private Map serviceHolder = new HashMap<>(4); + + public Builder selectBuilder(Integer key) { + return builderHolder.get(key); + } + + public DeduplicationService selectService(Integer key) { + return serviceHolder.get(key); + } + + public void putBuilder(Integer key, Builder builder) { + builderHolder.put(key, builder); + } + + public void putService(Integer key, DeduplicationService service) { + serviceHolder.put(key, service); + } +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java index 2db2503..abfc5d0 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java @@ -2,13 +2,11 @@ package com.java3y.austin.service.deduplication; import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.spring.annotation.ApolloConfig; -import com.google.common.collect.Lists; import com.java3y.austin.constant.AustinConstant; import com.java3y.austin.domain.DeduplicationParam; import com.java3y.austin.domain.TaskInfo; -import com.java3y.austin.service.deduplication.build.BuilderFactory; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; import java.util.List; @@ -21,39 +19,27 @@ import java.util.List; @Service public class DeduplicationRuleService { + public static final String DEDUPLICATION_RULE_KEY = "deduplication"; - private static final String SERVICE = "Service"; @ApolloConfig("boss.austin") private Config config; @Autowired - private ApplicationContext applicationContext; - @Autowired - private BuilderFactory builderFactory; - - //需要去重的服务 - private static final List DEDUPLICATION_LIST = Lists.newArrayList(DeduplicationConstants.CONTENT_DEDUPLICATION, DeduplicationConstants.FREQUENCY_DEDUPLICATION); + private DeduplicationHolder deduplicationHolder; public void duplication(TaskInfo taskInfo) { - // 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - String deduplication = config.getProperty(DeduplicationConstants.DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT); - //去重 - DEDUPLICATION_LIST.forEach( - key -> { - DeduplicationParam deduplicationParam = builderFactory.select(key).build(deduplication, key); - if (deduplicationParam != null) { - deduplicationParam.setTaskInfo(taskInfo); - DeduplicationService deduplicationService = findService(key + SERVICE); - deduplicationService.deduplication(deduplicationParam); - } - } - ); - + // 配置样例：{"deduplication_10":{"num":1,"time":300},"deduplication_20":{"num":5}} + String deduplicationConfig = config.getProperty(DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT); + + // 去重 + List deduplicationList = DeduplicationType.getDeduplicationList(); + for (Integer deduplicationType : deduplicationList) { + DeduplicationParam deduplicationParam = deduplicationHolder.selectBuilder(deduplicationType).build(deduplicationConfig, taskInfo); + if (deduplicationParam != null) { + deduplicationHolder.selectService(deduplicationType).deduplication(deduplicationParam); + } + } } - private DeduplicationService findService(String beanName) { - return applicationContext.getBean(beanName, DeduplicationService.class); - - } } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java new file mode 100644 index 0000000..62ade65 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java @@ -0,0 +1,40 @@ +package com.java3y.austin.service.deduplication.build; + +import com.alibaba.fastjson.JSONObject; +import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.service.deduplication.DeduplicationHolder; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.PostConstruct; + +/** + * @author 3y + * @date 2022/1/19 + */ +public abstract class AbstractDeduplicationBuilder implements Builder { + + protected Integer deduplicationType; + + @Autowired + private DeduplicationHolder deduplicationHolder; + + @PostConstruct + public void init() { + deduplicationHolder.putBuilder(deduplicationType, this); + } + + public DeduplicationParam getParamsFromConfig(Integer key, String duplicationConfig, TaskInfo taskInfo) { + JSONObject object = JSONObject.parseObject(duplicationConfig); + if (object == null) { + return null; + } + DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(CONFIG_PRE + key), DeduplicationParam.class); + if (deduplicationParam == null) { + return null; + } + deduplicationParam.setTaskInfo(taskInfo); + return deduplicationParam; + } + +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java index cd78e38..b2fe075 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java @@ -1,6 +1,7 @@ package com.java3y.austin.service.deduplication.build; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; /** * @author luohaojie @@ -8,5 +9,14 @@ import com.java3y.austin.domain.DeduplicationParam; */ public interface Builder { - DeduplicationParam build(String deduplication, String key); + String CONFIG_PRE = "deduplication_"; + + /** + * 根据配置构建去重参数 + * + * @param deduplication + * @param taskInfo + * @return + */ + DeduplicationParam build(String deduplication, TaskInfo taskInfo); } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java deleted file mode 100644 index 795dafa..0000000 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.java3y.austin.service.deduplication.build; - -import com.java3y.austin.service.deduplication.DeduplicationConstants; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - - -/** - * @author huskey - * @date 2022/1/18 - */ - -@Service -public class BuilderFactory { - - - private Map builderFactory = new HashMap<>(4); - - @Autowired - private Builder contentDeduplicationBuilder; - - @Autowired - private Builder frequencyDeduplicationBuilder; - - @PostConstruct - public void init() { - builderFactory.put(DeduplicationConstants.CONTENT_DEDUPLICATION, contentDeduplicationBuilder); - builderFactory.put(DeduplicationConstants.FREQUENCY_DEDUPLICATION, frequencyDeduplicationBuilder); - } - - - public Builder select(String key) { - return builderFactory.get(key); - } - -} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java index a542712..238819e 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java @@ -1,9 +1,9 @@ package com.java3y.austin.service.deduplication.build; -import com.alibaba.fastjson.JSONObject; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; import com.java3y.austin.enums.AnchorState; -import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; @@ -11,20 +11,20 @@ import org.springframework.stereotype.Service; * @author huskey * @date 2022/1/18 */ - @Service -public class ContentDeduplicationBuilder implements Builder { +public class ContentDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder { - public DeduplicationParam build(String deduplication, String key) { - JSONObject object = JSONObject.parseObject(deduplication); - if (object == null) { - return null; - } - DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); - if (deduplicationParam == null) { - return null; + public ContentDeduplicationBuilder() { + deduplicationType = DeduplicationType.CONTENT.getCode(); + } + + @Override + public DeduplicationParam build(String deduplication, TaskInfo taskInfo) { + DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo); + if (deduplication == null) { + return null; } - deduplicationParam.setAnchorState(AnchorState.RULE_DEDUPLICATION); + deduplicationParam.setAnchorState(AnchorState.CONTENT_DEDUPLICATION); return deduplicationParam; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java index 972ed87..b1d0ab4 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java @@ -1,10 +1,10 @@ package com.java3y.austin.service.deduplication.build; import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.JSONObject; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; import com.java3y.austin.enums.AnchorState; -import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; import java.util.Date; @@ -15,15 +15,14 @@ import java.util.Date; */ @Service -public class FrequencyDeduplicationBuilder implements Builder { +public class FrequencyDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder { + public FrequencyDeduplicationBuilder() { + deduplicationType = DeduplicationType.FREQUENCY.getCode(); + } @Override - public DeduplicationParam build(String deduplication, String key) { - JSONObject object = JSONObject.parseObject(deduplication); - if (object == null) { - return null; - } - DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); + public DeduplicationParam build(String deduplication, TaskInfo taskInfo) { + DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo); if (deduplicationParam == null) { return null; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java similarity index 89% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java index d1fb8c6..b51b9db 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java @@ -1,16 +1,17 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.core.collection.CollUtil; import com.java3y.austin.constant.AustinConstant; import com.java3y.austin.domain.AnchorInfo; import com.java3y.austin.domain.DeduplicationParam; import com.java3y.austin.domain.TaskInfo; -import com.java3y.austin.enums.AnchorState; +import com.java3y.austin.service.deduplication.DeduplicationHolder; import com.java3y.austin.utils.LogUtils; import com.java3y.austin.utils.RedisUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import javax.annotation.PostConstruct; import java.util.*; /** @@ -20,10 +21,20 @@ import java.util.*; */ @Slf4j public abstract class AbstractDeduplicationService implements DeduplicationService { + protected Integer deduplicationType; + + @Autowired + private DeduplicationHolder deduplicationHolder; + + @PostConstruct + private void init() { + deduplicationHolder.putService(deduplicationType, this); + } @Autowired private RedisUtils redisUtils; + @Override public void deduplication(DeduplicationParam param) { TaskInfo taskInfo = param.getTaskInfo(); Set filterReceiver = new HashSet<>(taskInfo.getReceiver().size()); diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java similarity index 79% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java index 1b41e44..a305359 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java @@ -1,8 +1,9 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.crypto.digest.DigestUtil; import com.alibaba.fastjson.JSON; import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; /** @@ -13,6 +14,10 @@ import org.springframework.stereotype.Service; @Service public class ContentDeduplicationService extends AbstractDeduplicationService { + public ContentDeduplicationService() { + deduplicationType = DeduplicationType.CONTENT.getCode(); + } + /** * 内容去重 构建key *

diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java similarity index 64% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java index 35c4b80..e0abf27 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java @@ -1,4 +1,4 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import com.java3y.austin.domain.DeduplicationParam; @@ -8,5 +8,9 @@ import com.java3y.austin.domain.DeduplicationParam; */ public interface DeduplicationService { + /** + * 去重 + * @param param + */ void deduplication(DeduplicationParam param); } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java similarity index 80% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java index c135f46..bc23513 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java @@ -1,7 +1,8 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.core.util.StrUtil; import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; /** @@ -12,6 +13,10 @@ import org.springframework.stereotype.Service; @Service public class FrequencyDeduplicationService extends AbstractDeduplicationService { + public FrequencyDeduplicationService() { + deduplicationType = DeduplicationType.FREQUENCY.getCode(); + } + private static final String PREFIX = "FRE"; /**