diff --git a/pom.xml b/pom.xml index 897026e..2665f73 100644 --- a/pom.xml +++ b/pom.xml @@ -288,6 +288,22 @@ easyexcel 3.3.4 --> + + + us.codecraft + webmagic-core + 1.0.0 + + + us.codecraft + webmagic-extension + 1.0.0 + + + com.microsoft.playwright + playwright + 1.47.0 + @@ -329,7 +345,6 @@ - diff --git a/src/main/java/jj/tech/paolu/Application.java b/src/main/java/jj/tech/paolu/Application.java index 9c5447f..4a8e544 100644 --- a/src/main/java/jj/tech/paolu/Application.java +++ b/src/main/java/jj/tech/paolu/Application.java @@ -9,6 +9,9 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.scheduling.annotation.EnableScheduling; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; + /** * @SpringBootApplication 是一个方便的注释, 它添加了以下所有内容: @@ -43,6 +46,8 @@ public class Application { // public static void main(String[] args) { // SpringApplication.run(Application.class, args); // } + + // called OK on docker "start " public static void main(String[] args) { diff --git a/src/main/java/jj/tech/paolu/biz/webadmin/component/Hgt8Component.java b/src/main/java/jj/tech/paolu/biz/webadmin/component/Hgt8Component.java new file mode 100644 index 0000000..bb66fac --- /dev/null +++ b/src/main/java/jj/tech/paolu/biz/webadmin/component/Hgt8Component.java @@ -0,0 +1,102 @@ +package jj.tech.paolu.biz.webadmin.component; + +import org.springframework.stereotype.Component; + +import com.microsoft.playwright.APIRequest; +import com.microsoft.playwright.APIRequestContext; +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.BrowserType; +import com.microsoft.playwright.Locator; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import com.microsoft.playwright.options.AriaRole; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +@Component +public class Hgt8Component { + + public Playwright playwright; + public Browser browser; + public BrowserContext browserContext; + public APIRequestContext aPIRequestContext; + public Page page; + + + + @PostConstruct + public void start() throws Exception { + + this.playwright = Playwright.create(); + this.playwright.selectors().setTestIdAttribute("id"); + + this.browser = playwright.chromium().launch( + new BrowserType.LaunchOptions() + .setHeadless(false) + ); + + this.browserContext = browser.newContext(); + this.browserContext.setDefaultNavigationTimeout(5000); + + + this.aPIRequestContext = browserContext.request(); + //browserContext.newPage(); + + this.page = browserContext.newPage(); + + + + } + + @PreDestroy + public void stop() { +// page.close(); + browser.close(); + playwright.close(); + System.out.println("Hgt8 is close"); + } + + + public static void main(String[] args) { + try { + Playwright playwright = Playwright.create(); + Browser browser = playwright.chromium().launch( + new BrowserType.LaunchOptions().setHeadless(false) + ); + Page page = browser.newPage(); +// page.navigate("https://sse8.idc9998.com"); + page.navigate("https://sol8.idc9998.com"); + + + + // Expect a title "to contain" a substring. + + + // create a locator + Locator getStarted = page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Get Started")); + + // Expect an attribute "to be strictly equal" to the value. +// assertThat(getStarted).hasAttribute("href", "/docs/intro"); + + // Click the get started link. +// getStarted.click(); + +// assertThat(browser.close()).hasAttribute("href", "/docs/intro"); + +// page.pause(); + + // Expects page to have a heading with the name of Installation. +// assertThat(page.getByRole(AriaRole.HEADING, +// new Page.GetByRoleOptions().setName("Installation"))).isVisible(); + page.close(); + browser.close(); + playwright.close(); + + + }catch (Exception e) { + // TODO: handle exception + } + } + +} diff --git a/src/main/java/jj/tech/paolu/biz/webadmin/controller/Hgt8Controller.java b/src/main/java/jj/tech/paolu/biz/webadmin/controller/Hgt8Controller.java new file mode 100644 index 0000000..82cd863 --- /dev/null +++ b/src/main/java/jj/tech/paolu/biz/webadmin/controller/Hgt8Controller.java @@ -0,0 +1,113 @@ +package jj.tech.paolu.biz.webadmin.controller; + +import static com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat; + +import java.util.HashMap; + +import org.apache.commons.codec.binary.Base64; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.JsonNode; +import com.microsoft.playwright.APIResponse; +import com.microsoft.playwright.FrameLocator; +import com.microsoft.playwright.Response; +import com.microsoft.playwright.options.RequestOptions; + +import io.swagger.v3.oas.annotations.Operation; +import jj.tech.paolu.biz.webadmin.component.Hgt8Component; +import jj.tech.paolu.biz.webadmin.vo.YmCodeVo; +import jj.tech.paolu.utils.R; +import jj.tech.paolu.utils.YMHttp; + +@RestController +@RequestMapping(value = "/op/hgt8",name = "导航到hgt8") +public class Hgt8Controller { + @Autowired + public Hgt8Component hgt8Component; + + + @Operation(summary = "test", description = "test") + @PostMapping("/test") + public Object login(@Validated @RequestBody YmCodeVo prame) { +// hgt8Component.page.navigate("https://sol8.idc9998.com/search.aspx"); +// hgt8Component.page.locator("xpath=//iframe[@id=lineCode]").fill("bk7897"); +// hgt8Component.page.getByTestId("lineCode").fill("bk7897"); + + hgt8Component.page.navigate("https://sol8.idc9998.com"); + hgt8Component.page.frameLocator("#ifr_main") + .getByTestId("lineCode") + .fill("339hp"); + + hgt8Component.page.frameLocator("#ifr_main") + .locator("xpath=//input[@class='btn']") + .click(); + + Response response = hgt8Component.page.waitForResponse("**/captcha_image/2*", () -> { + + }); + byte[] image_bytes = response.body(); + + JsonNode root = + YMHttp.customApi(Base64.encodeBase64String(image_bytes)); + JsonNode code_num = root.get("data").get("data"); + + //System.err.println(code_num.asText()); + FrameLocator ifr_main = hgt8Component.page.frameLocator("#ifr_main"); + + + ifr_main.getByTestId("userName") + .fill("bk7897"); + + ifr_main.getByTestId("password") + .fill("asas12123"); + + ifr_main.getByTestId("code") + .fill(code_num.asText()); + + ifr_main.getByTestId("btnSubmit") + .click(); + + + +// assertThat(hgt8Component.page.frameLocator("#ifr_main") +// .locator("xpath=//input[@class='protocol-btn-y']")).is; + + //同意 + hgt8Component.page.frameLocator("#ifr_main") + .locator("xpath=//input[@class='protocol-btn-y']") + .click(); + +// ifr_main.getByTestId("tc_Close") +// .click(); + ifr_main.getByText("历史开奖") + .click(); +// Assert.assertEquals(apiPostResponse.status(), 201); + +// {"gameID":"G_24","currtentPage":1,"asDate":"2024-09-23","issuet":""} + HashMap pram = new HashMap(); + pram.put("gameID", "G_24"); + pram.put("currtentPage","1"); + pram.put("asDate","2024-09-23"); + pram.put("issuet", ""); + + RequestOptions requestOptions = + RequestOptions.create().setHeader("Content-Type", "application/json") + .setData(pram); + + APIResponse addNewPet = + hgt8Component.aPIRequestContext.post("https://sol8.idc9998.com/wb2/jg0001/s001s", requestOptions); + + System.err.println(addNewPet.text()); + + + + //ifr_main.getByTestId("captcha"); + + return R.SUCCESS("ok"); + } +} diff --git a/src/main/java/jj/tech/paolu/biz/webadmin/controller/YMcodeController.java b/src/main/java/jj/tech/paolu/biz/webadmin/controller/YMcodeController.java index 549f93d..b50ba26 100644 --- a/src/main/java/jj/tech/paolu/biz/webadmin/controller/YMcodeController.java +++ b/src/main/java/jj/tech/paolu/biz/webadmin/controller/YMcodeController.java @@ -20,10 +20,7 @@ public class YMcodeController { @Operation(summary = "需要识别图片的base64字符串", description = "image_base64") @PostMapping("/code") public Object login(@Validated @RequestBody YmCodeVo prame) { - - Map map = YMHttp.customApi(prame.image_base64); - - return R.SUCCESS(map); + return R.SUCCESS(YMHttp.customApi(prame.image_base64)); } diff --git a/src/main/java/jj/tech/paolu/utils/YMHttp.java b/src/main/java/jj/tech/paolu/utils/YMHttp.java index df6c654..7804161 100644 --- a/src/main/java/jj/tech/paolu/utils/YMHttp.java +++ b/src/main/java/jj/tech/paolu/utils/YMHttp.java @@ -6,13 +6,41 @@ import java.util.Map; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; public class YMHttp { + public static ObjectMapper objectMapper = new ObjectMapper(); public static String ym_url = "http://api.jfbym.com/api/YmServer/customApi"; - public static Map customApi(String image_base64){ +// public static Map customApi(String image_base64){ +// try { +// Map param = new HashMap<>(); +// param.put("token", "Cx08lMubmQauvOOhD0rp_qdnebs--Hi7bPdy9No6Uvw"); +// param.put("type", "10110"); +// param.put("image", image_base64); +// +// String stean = Request.Post(ym_url) +// .connectTimeout(2500) +// .socketTimeout(2500) +// .bodyString(objectMapper.writeValueAsString(param), ContentType.APPLICATION_JSON) +// .execute() +// .returnContent() +// .asString(); +// +// @SuppressWarnings("unchecked") +// HashMap jsonObject = objectMapper.readValue(stean, HashMap.class); +// +// return jsonObject; +// +// } catch (Exception e) { +// e.printStackTrace(); +// return null; +// } +// } + + public static JsonNode customApi(String image_base64){ try { Map param = new HashMap<>(); param.put("token", "Cx08lMubmQauvOOhD0rp_qdnebs--Hi7bPdy9No6Uvw"); @@ -27,16 +55,12 @@ public class YMHttp { .returnContent() .asString(); - @SuppressWarnings("unchecked") - HashMap jsonObject = objectMapper.readValue(stean, HashMap.class); - - return jsonObject; + return objectMapper.readTree(stean); } catch (Exception e) { e.printStackTrace(); + return null; } - - return null; } public static void main(String[] args) throws Exception{ @@ -63,9 +87,9 @@ public class YMHttp { // // } - Map r = - YMHttp.customApi("ss"); - System.out.println(r); +// Map r = +// YMHttp.customApi("ss"); +// System.out.println(r); } diff --git a/src/main/java/jj/tech/paolu/utils/crawler/Hgt8Processor.java b/src/main/java/jj/tech/paolu/utils/crawler/Hgt8Processor.java new file mode 100644 index 0000000..194091d --- /dev/null +++ b/src/main/java/jj/tech/paolu/utils/crawler/Hgt8Processor.java @@ -0,0 +1,151 @@ +package jj.tech.paolu.utils.crawler; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import javax.imageio.ImageIO; + +import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpEntity; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.apache.http.util.EntityUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jj.tech.paolu.utils.YMHttp; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; + +public class Hgt8Processor implements PageProcessor { + private Site site = Site.me().setRetryTimes(3).setSleepTime(100); + + @Override + public Site getSite() { + return site; + } + + @Override + public void process(Page page) { + page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all()); + page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString()); + page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString()); + if (page.getResultItems().get("name")==null){ + //skip this page + page.setSkip(true); + } + page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()")); + } + + public static void main(String[] args) throws Exception{ + + + +// Spider.create(new Hgt8Processor()).addUrl("https://sse8.idc9998.com").thread(1).run(); + + +// Map param = new HashMap<>(); +// param.put("lineCode", "be358"); +// param.put("isSaveCode", "0"); +// ObjectMapper objectMapper = new ObjectMapper(); +// +// String stean = Request.Post("https://sse8.idc9998.com") +// +// .connectTimeout(2500) +// .socketTimeout(2500) +// .bodyString(objectMapper.writeValueAsString(param), ContentType.APPLICATION_JSON) +// .execute() +// .returnContent() +// .asString(); +// System.err.println(stean); + + + try { + + BasicCookieStore cookieStore = new BasicCookieStore(); + BasicClientCookie cookie = new BasicClientCookie("name", "vale"); + cookie.setDomain("idc9998.com"); + cookie.setPath("cookie"); + cookieStore.addCookie(cookie); + + + CloseableHttpClient httpclient = HttpClients.custom() + .setDefaultCookieStore(cookieStore) + .build(); + + + ObjectMapper objectMapper = new ObjectMapper(); + Map param = new HashMap<>(); + param.put("lineCode", "be358"); + param.put("isSaveCode", "0"); + String paramString = objectMapper.writeValueAsString(param); + + HttpPost httpPost = new HttpPost("https://sse8.idc9998.com"); + + + StringEntity paramEntity = new StringEntity(paramString, ContentType.APPLICATION_JSON); + httpPost.setEntity(paramEntity); + + + CloseableHttpResponse response = httpclient + .execute(httpPost); + + HttpEntity respoetEntity = response.getEntity(); + String responseString = EntityUtils.toString(respoetEntity, "UTF-8"); + + System.out.println(responseString); + + + + + HttpPost httpPost1 = new HttpPost("https://sse8.idc9998.com/captcha_image/2?d="+""); + httpPost1.addHeader("content-type", ContentType.IMAGE_TIFF.toString()); + + + CloseableHttpResponse response1 = httpclient + .execute(httpPost1); + + //System.err.println(EntityUtils.toString(response1.getEntity(), "UTF-8")); + HttpEntity entity = response1.getEntity(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + entity.writeTo(byteArrayOutputStream); + + String image = + Base64.encodeBase64String(byteArrayOutputStream.toByteArray()); + + System.err.println(image); + +// JsonNode jsonNode = +// YMHttp.customApi(image); +// System.out.println(jsonNode); + + + response.close(); + response1.close(); + + }catch (Exception e) { + System.out.println(e); + } + + + + + } + +} diff --git a/src/main/java/jj/tech/paolu/utils/playwright/Hgt8.java b/src/main/java/jj/tech/paolu/utils/playwright/Hgt8.java new file mode 100644 index 0000000..eff4a4f --- /dev/null +++ b/src/main/java/jj/tech/paolu/utils/playwright/Hgt8.java @@ -0,0 +1,80 @@ +//package jj.tech.paolu.utils.playwright; +// +//import static com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat; +// +//import org.springframework.stereotype.Component; +// +//import com.microsoft.playwright.Browser; +//import com.microsoft.playwright.BrowserType; +//import com.microsoft.playwright.Locator; +//import com.microsoft.playwright.Page; +//import com.microsoft.playwright.Playwright; +//import com.microsoft.playwright.options.AriaRole; +// +//import jakarta.annotation.PostConstruct; +//import jakarta.annotation.PreDestroy; +//@Component +//public class Hgt8 { +// public Playwright playwright; +// public Browser browser; +// public Page page; +// +// @PostConstruct +// public void start() throws Exception { +// this.playwright = Playwright.create(); +// this.browser = playwright.chromium().launch( +// new BrowserType.LaunchOptions().setHeadless(false) +// ); +// page = browser.newPage(); +// } +// +// @PreDestroy +// public void stop() { +// page.close(); +// browser.close(); +// playwright.close(); +// System.out.println("Hgt8 is close"); +// } +// +// +// public static void main(String[] args) { +// try { +// Playwright playwright = Playwright.create(); +// Browser browser = playwright.chromium().launch( +// new BrowserType.LaunchOptions().setHeadless(false) +// ); +// Page page = browser.newPage(); +//// page.navigate("https://sse8.idc9998.com"); +// page.navigate("https://sol8.idc9998.com"); +// +// +// // Expect a title "to contain" a substring. +// +// +// // create a locator +// Locator getStarted = page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Get Started")); +// +// // Expect an attribute "to be strictly equal" to the value. +//// assertThat(getStarted).hasAttribute("href", "/docs/intro"); +// +// // Click the get started link. +//// getStarted.click(); +// +//// assertThat(browser.close()).hasAttribute("href", "/docs/intro"); +// +//// page.pause(); +// +// // Expects page to have a heading with the name of Installation. +//// assertThat(page.getByRole(AriaRole.HEADING, +//// new Page.GetByRoleOptions().setName("Installation"))).isVisible(); +// page.close(); +// browser.close(); +// playwright.close(); +// +// +// }catch (Exception e) { +// // TODO: handle exception +// } +// } +// +//} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b1b0f07..dd13ae2 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -98,49 +98,4 @@ logging: org.mybatis: info org.mybatis.dynamic.sql: debug jj.tech.paolu: info - org.fisco.bcos.sdk: error - -#bcn -### Required, node's {ip:port} to connect. -system: - peers: 192.168.0.180:20200 - ### Required - groupId: 1 - ### Optional. Default will search conf,config,src/main/conf/src/main/config - certPath: conf,src/main/resources/conf - ### Optional. If don't specify a random private key will be used - hexPrivateKey: '179b59aec0b9ea33ce6bcdc82a5fd343199bcd0be129ffe1d047a015c3dd0166' - blockChainTablePrefix: 'SZTD_0611_1' - blockChainProviderName: '深中通道-开发环境0611_1' - syn2BlockConcurrentThreadCount: 100 #向区块链同步目录数据时的并发线程数，区块链服务器性能越好，此值可越高 - ### Optional. Please fill this address if you want to use related service - contract: - applicationAddress: '0x0685f9ec997695f3168e45363a2070fe7579426b' - ### Optional. Please fill this address if you want to use related service - fileAddress: '0x108d01a27e5426e28af3e0c237f8605ce92ee43e' - ### Optional. Please fill this address if you want to use related service - archivesFileAddress: '0x3a78f6ef2c8729cfd36a50fd9a233498b5fdfa3b' - ### Optional. Please fill this address if you want to use related service - catalogAddress: '0xadc69ae2af98cb20dadadbe106fccb4412f1c750' - ### Optional. Please fill this address if you want to use related service - providerAddress: '0x8b97b1a800789a0c060edc98a25a6ae8e88fcf53' - readingRecordAddress: '0x9174823be980132fd6e14f5d74294c5830561f1d' - decryptionRecordAddress: '0x790143aa4a2f1162b81f343c825b3d42a101efc7' - downloadRecordAddress: '0x70296f76c871599cfb9940a7550f53aba555be00' - certRecordAddress: '0x04106399e9d317d4cb0dc606a7201e203e1cffe7' - createTableAddress: '0x1e5cbd200cb82fec444406772153de22a3502ba3' - utilsAddress: '0xd0a601e191178f44e7f226d0ab4124ad440dfff2' - orgCertAddress: '0x1a6d91b6a163040ca76fbf860fa968dbff35a264' - personCertAddress: '0x4ab8b3e641774802217545295482272ecaa1ef54' - -weibai: - appKey: SZTD00898 - appSecret: 16E58F6750C246818FBD84EAEF3FC998 - #getSortTreeUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getSortTree - #getCatalogUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getCatalog - #getAnnexUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getAnnex - #getSortSubUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getSortSub - getSortTreeUrl: http://127.0.0.1:8083/op/wb/getSortTree - getCatalogUrl: http://127.0.0.1:8083/op/wb/getCatalog - getAnnexUrl: http://127.0.0.1:8083/op/wb/getAnnex - getSortSubUrl: http://127.0.0.1:8083/op/wb/getSortSub + org.fisco.bcos.sdk: error \ No newline at end of file