http
All checks were successful
/ build-image (push) Successful in 11m16s

This commit is contained in:
d 2024-09-23 16:14:48 +08:00
parent 02eb8a4e46
commit cf25ac0581
9 changed files with 505 additions and 63 deletions

21
pom.xml
View File

@ -289,6 +289,22 @@
<version>3.3.4</version>
</dependency>-->
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>1.47.0</version>
</dependency>
@ -329,7 +345,6 @@
</executions>
</plugin>
<!--<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>sql-maven-plugin</artifactId>
@ -556,7 +571,7 @@
</repositories>
<pluginRepositories>
<!--<pluginRepositories>
<pluginRepository>
<id>aliyun-plugin</id>
<name>aliyun-plugin</name>
@ -588,7 +603,7 @@
<id>spring-milestones</id>
<url>https://repo.spring.io/milestone</url>
</pluginRepository>
</pluginRepositories>
</pluginRepositories>-->
</project>

View File

@ -9,6 +9,9 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
/**
* @SpringBootApplication 是一个方便的注释, 它添加了以下所有内容:
@ -44,6 +47,8 @@ public class Application {
// SpringApplication.run(Application.class, args);
// }
// called OK on docker "start <containerId>"
public static void main(String[] args) {
SpringApplication.run(Application.class, args);

View File

@ -0,0 +1,102 @@
package jj.tech.paolu.biz.webadmin.component;
import org.springframework.stereotype.Component;
import com.microsoft.playwright.APIRequest;
import com.microsoft.playwright.APIRequestContext;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.BrowserContext;
import com.microsoft.playwright.BrowserType;
import com.microsoft.playwright.Locator;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.Playwright;
import com.microsoft.playwright.options.AriaRole;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
@Component
public class Hgt8Component {
public Playwright playwright;
public Browser browser;
public BrowserContext browserContext;
public APIRequestContext aPIRequestContext;
public Page page;
@PostConstruct
public void start() throws Exception {
this.playwright = Playwright.create();
this.playwright.selectors().setTestIdAttribute("id");
this.browser = playwright.chromium().launch(
new BrowserType.LaunchOptions()
.setHeadless(false)
);
this.browserContext = browser.newContext();
this.browserContext.setDefaultNavigationTimeout(5000);
this.aPIRequestContext = browserContext.request();
//browserContext.newPage();
this.page = browserContext.newPage();
}
@PreDestroy
public void stop() {
// page.close();
browser.close();
playwright.close();
System.out.println("Hgt8 is close");
}
public static void main(String[] args) {
try {
Playwright playwright = Playwright.create();
Browser browser = playwright.chromium().launch(
new BrowserType.LaunchOptions().setHeadless(false)
);
Page page = browser.newPage();
// page.navigate("https://sse8.idc9998.com");
page.navigate("https://sol8.idc9998.com");
// Expect a title "to contain" a substring.
// create a locator
Locator getStarted = page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Get Started"));
// Expect an attribute "to be strictly equal" to the value.
// assertThat(getStarted).hasAttribute("href", "/docs/intro");
// Click the get started link.
// getStarted.click();
// assertThat(browser.close()).hasAttribute("href", "/docs/intro");
// page.pause();
// Expects page to have a heading with the name of Installation.
// assertThat(page.getByRole(AriaRole.HEADING,
// new Page.GetByRoleOptions().setName("Installation"))).isVisible();
page.close();
browser.close();
playwright.close();
}catch (Exception e) {
// TODO: handle exception
}
}
}

View File

@ -0,0 +1,113 @@
package jj.tech.paolu.biz.webadmin.controller;
import static com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat;
import java.util.HashMap;
import org.apache.commons.codec.binary.Base64;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.JsonNode;
import com.microsoft.playwright.APIResponse;
import com.microsoft.playwright.FrameLocator;
import com.microsoft.playwright.Response;
import com.microsoft.playwright.options.RequestOptions;
import io.swagger.v3.oas.annotations.Operation;
import jj.tech.paolu.biz.webadmin.component.Hgt8Component;
import jj.tech.paolu.biz.webadmin.vo.YmCodeVo;
import jj.tech.paolu.utils.R;
import jj.tech.paolu.utils.YMHttp;
@RestController
@RequestMapping(value = "/op/hgt8",name = "导航到hgt8")
public class Hgt8Controller {
@Autowired
public Hgt8Component hgt8Component;
@Operation(summary = "test", description = "test")
@PostMapping("/test")
public Object login(@Validated @RequestBody YmCodeVo prame) {
// hgt8Component.page.navigate("https://sol8.idc9998.com/search.aspx");
// hgt8Component.page.locator("xpath=//iframe[@id=lineCode]").fill("bk7897");
// hgt8Component.page.getByTestId("lineCode").fill("bk7897");
hgt8Component.page.navigate("https://sol8.idc9998.com");
hgt8Component.page.frameLocator("#ifr_main")
.getByTestId("lineCode")
.fill("339hp");
hgt8Component.page.frameLocator("#ifr_main")
.locator("xpath=//input[@class='btn']")
.click();
Response response = hgt8Component.page.waitForResponse("**/captcha_image/2*", () -> {
});
byte[] image_bytes = response.body();
JsonNode root =
YMHttp.customApi(Base64.encodeBase64String(image_bytes));
JsonNode code_num = root.get("data").get("data");
//System.err.println(code_num.asText());
FrameLocator ifr_main = hgt8Component.page.frameLocator("#ifr_main");
ifr_main.getByTestId("userName")
.fill("bk7897");
ifr_main.getByTestId("password")
.fill("asas12123");
ifr_main.getByTestId("code")
.fill(code_num.asText());
ifr_main.getByTestId("btnSubmit")
.click();
// assertThat(hgt8Component.page.frameLocator("#ifr_main")
// .locator("xpath=//input[@class='protocol-btn-y']")).is;
//同意
hgt8Component.page.frameLocator("#ifr_main")
.locator("xpath=//input[@class='protocol-btn-y']")
.click();
// ifr_main.getByTestId("tc_Close")
// .click();
ifr_main.getByText("历史开奖")
.click();
// Assert.assertEquals(apiPostResponse.status(), 201);
// {"gameID":"G_24","currtentPage":1,"asDate":"2024-09-23","issuet":""}
HashMap<String, String> pram = new HashMap<String, String>();
pram.put("gameID", "G_24");
pram.put("currtentPage","1");
pram.put("asDate","2024-09-23");
pram.put("issuet", "");
RequestOptions requestOptions =
RequestOptions.create().setHeader("Content-Type", "application/json")
.setData(pram);
APIResponse addNewPet =
hgt8Component.aPIRequestContext.post("https://sol8.idc9998.com/wb2/jg0001/s001s", requestOptions);
System.err.println(addNewPet.text());
//ifr_main.getByTestId("captcha");
return R.SUCCESS("ok");
}
}

View File

@ -20,10 +20,7 @@ public class YMcodeController {
@Operation(summary = "需要识别图片的base64字符串", description = "image_base64")
@PostMapping("/code")
public Object login(@Validated @RequestBody YmCodeVo prame) {
Map<String, Object> map = YMHttp.customApi(prame.image_base64);
return R.SUCCESS(map);
return R.SUCCESS(YMHttp.customApi(prame.image_base64));
}

View File

@ -6,13 +6,41 @@ import java.util.Map;
import org.apache.http.client.fluent.Request;
import org.apache.http.entity.ContentType;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class YMHttp {
public static ObjectMapper objectMapper = new ObjectMapper();
public static String ym_url = "http://api.jfbym.com/api/YmServer/customApi";
public static Map<String,Object> customApi(String image_base64){
// public static Map<String,Object> customApi(String image_base64){
// try {
// Map<String, Object> param = new HashMap<>();
// param.put("token", "Cx08lMubmQauvOOhD0rp_qdnebs--Hi7bPdy9No6Uvw");
// param.put("type", "10110");
// param.put("image", image_base64);
//
// String stean = Request.Post(ym_url)
// .connectTimeout(2500)
// .socketTimeout(2500)
// .bodyString(objectMapper.writeValueAsString(param), ContentType.APPLICATION_JSON)
// .execute()
// .returnContent()
// .asString();
//
// @SuppressWarnings("unchecked")
// HashMap<String, Object> jsonObject = objectMapper.readValue(stean, HashMap.class);
//
// return jsonObject;
//
// } catch (Exception e) {
// e.printStackTrace();
// return null;
// }
// }
public static JsonNode customApi(String image_base64){
try {
Map<String, Object> param = new HashMap<>();
param.put("token", "Cx08lMubmQauvOOhD0rp_qdnebs--Hi7bPdy9No6Uvw");
@ -27,16 +55,12 @@ public class YMHttp {
.returnContent()
.asString();
@SuppressWarnings("unchecked")
HashMap<String, Object> jsonObject = objectMapper.readValue(stean, HashMap.class);
return jsonObject;
return objectMapper.readTree(stean);
} catch (Exception e) {
e.printStackTrace();
return null;
}
return null;
}
public static void main(String[] args) throws Exception{
@ -63,9 +87,9 @@ public class YMHttp {
//
// }
Map<String,Object> r =
YMHttp.customApi("ss");
System.out.println(r);
// Map<String,Object> r =
// YMHttp.customApi("ss");
// System.out.println(r);
}

View File

@ -0,0 +1,151 @@
package jj.tech.paolu.utils.crawler;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.imageio.ImageIO;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.fluent.Executor;
import org.apache.http.client.fluent.Request;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.util.EntityUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jj.tech.paolu.utils.YMHttp;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
public class Hgt8Processor implements PageProcessor {
private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
@Override
public Site getSite() {
return site;
}
@Override
public void process(Page page) {
page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
if (page.getResultItems().get("name")==null){
//skip this page
page.setSkip(true);
}
page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
}
public static void main(String[] args) throws Exception{
// Spider.create(new Hgt8Processor()).addUrl("https://sse8.idc9998.com").thread(1).run();
// Map<String, Object> param = new HashMap<>();
// param.put("lineCode", "be358");
// param.put("isSaveCode", "0");
// ObjectMapper objectMapper = new ObjectMapper();
//
// String stean = Request.Post("https://sse8.idc9998.com")
//
// .connectTimeout(2500)
// .socketTimeout(2500)
// .bodyString(objectMapper.writeValueAsString(param), ContentType.APPLICATION_JSON)
// .execute()
// .returnContent()
// .asString();
// System.err.println(stean);
try {
BasicCookieStore cookieStore = new BasicCookieStore();
BasicClientCookie cookie = new BasicClientCookie("name", "vale");
cookie.setDomain("idc9998.com");
cookie.setPath("cookie");
cookieStore.addCookie(cookie);
CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCookieStore(cookieStore)
.build();
ObjectMapper objectMapper = new ObjectMapper();
Map<String, Object> param = new HashMap<>();
param.put("lineCode", "be358");
param.put("isSaveCode", "0");
String paramString = objectMapper.writeValueAsString(param);
HttpPost httpPost = new HttpPost("https://sse8.idc9998.com");
StringEntity paramEntity = new StringEntity(paramString, ContentType.APPLICATION_JSON);
httpPost.setEntity(paramEntity);
CloseableHttpResponse response = httpclient
.execute(httpPost);
HttpEntity respoetEntity = response.getEntity();
String responseString = EntityUtils.toString(respoetEntity, "UTF-8");
System.out.println(responseString);
HttpPost httpPost1 = new HttpPost("https://sse8.idc9998.com/captcha_image/2?d="+"");
httpPost1.addHeader("content-type", ContentType.IMAGE_TIFF.toString());
CloseableHttpResponse response1 = httpclient
.execute(httpPost1);
//System.err.println(EntityUtils.toString(response1.getEntity(), "UTF-8"));
HttpEntity entity = response1.getEntity();
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
entity.writeTo(byteArrayOutputStream);
String image =
Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
System.err.println(image);
// JsonNode jsonNode =
// YMHttp.customApi(image);
// System.out.println(jsonNode);
response.close();
response1.close();
}catch (Exception e) {
System.out.println(e);
}
}
}

View File

@ -0,0 +1,80 @@
//package jj.tech.paolu.utils.playwright;
//
//import static com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat;
//
//import org.springframework.stereotype.Component;
//
//import com.microsoft.playwright.Browser;
//import com.microsoft.playwright.BrowserType;
//import com.microsoft.playwright.Locator;
//import com.microsoft.playwright.Page;
//import com.microsoft.playwright.Playwright;
//import com.microsoft.playwright.options.AriaRole;
//
//import jakarta.annotation.PostConstruct;
//import jakarta.annotation.PreDestroy;
//@Component
//public class Hgt8 {
// public Playwright playwright;
// public Browser browser;
// public Page page;
//
// @PostConstruct
// public void start() throws Exception {
// this.playwright = Playwright.create();
// this.browser = playwright.chromium().launch(
// new BrowserType.LaunchOptions().setHeadless(false)
// );
// page = browser.newPage();
// }
//
// @PreDestroy
// public void stop() {
// page.close();
// browser.close();
// playwright.close();
// System.out.println("Hgt8 is close");
// }
//
//
// public static void main(String[] args) {
// try {
// Playwright playwright = Playwright.create();
// Browser browser = playwright.chromium().launch(
// new BrowserType.LaunchOptions().setHeadless(false)
// );
// Page page = browser.newPage();
//// page.navigate("https://sse8.idc9998.com");
// page.navigate("https://sol8.idc9998.com");
//
//
// // Expect a title "to contain" a substring.
//
//
// // create a locator
// Locator getStarted = page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Get Started"));
//
// // Expect an attribute "to be strictly equal" to the value.
//// assertThat(getStarted).hasAttribute("href", "/docs/intro");
//
// // Click the get started link.
//// getStarted.click();
//
//// assertThat(browser.close()).hasAttribute("href", "/docs/intro");
//
//// page.pause();
//
// // Expects page to have a heading with the name of Installation.
//// assertThat(page.getByRole(AriaRole.HEADING,
//// new Page.GetByRoleOptions().setName("Installation"))).isVisible();
// page.close();
// browser.close();
// playwright.close();
//
//
// }catch (Exception e) {
// // TODO: handle exception
// }
// }
//
//}

View File

@ -99,48 +99,3 @@ logging:
org.mybatis.dynamic.sql: debug
jj.tech.paolu: info
org.fisco.bcos.sdk: error
#bcn
### Required, node's {ip:port} to connect.
system:
peers: 192.168.0.180:20200
### Required
groupId: 1
### Optional. Default will search conf,config,src/main/conf/src/main/config
certPath: conf,src/main/resources/conf
### Optional. If don't specify a random private key will be used
hexPrivateKey: '179b59aec0b9ea33ce6bcdc82a5fd343199bcd0be129ffe1d047a015c3dd0166'
blockChainTablePrefix: 'SZTD_0611_1'
blockChainProviderName: '深中通道-开发环境0611_1'
syn2BlockConcurrentThreadCount: 100 #向区块链同步目录数据时的并发线程数,区块链服务器性能越好,此值可越高
### Optional. Please fill this address if you want to use related service
contract:
applicationAddress: '0x0685f9ec997695f3168e45363a2070fe7579426b'
### Optional. Please fill this address if you want to use related service
fileAddress: '0x108d01a27e5426e28af3e0c237f8605ce92ee43e'
### Optional. Please fill this address if you want to use related service
archivesFileAddress: '0x3a78f6ef2c8729cfd36a50fd9a233498b5fdfa3b'
### Optional. Please fill this address if you want to use related service
catalogAddress: '0xadc69ae2af98cb20dadadbe106fccb4412f1c750'
### Optional. Please fill this address if you want to use related service
providerAddress: '0x8b97b1a800789a0c060edc98a25a6ae8e88fcf53'
readingRecordAddress: '0x9174823be980132fd6e14f5d74294c5830561f1d'
decryptionRecordAddress: '0x790143aa4a2f1162b81f343c825b3d42a101efc7'
downloadRecordAddress: '0x70296f76c871599cfb9940a7550f53aba555be00'
certRecordAddress: '0x04106399e9d317d4cb0dc606a7201e203e1cffe7'
createTableAddress: '0x1e5cbd200cb82fec444406772153de22a3502ba3'
utilsAddress: '0xd0a601e191178f44e7f226d0ab4124ad440dfff2'
orgCertAddress: '0x1a6d91b6a163040ca76fbf860fa968dbff35a264'
personCertAddress: '0x4ab8b3e641774802217545295482272ecaa1ef54'
weibai:
appKey: SZTD00898
appSecret: 16E58F6750C246818FBD84EAEF3FC998
#getSortTreeUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getSortTree
#getCatalogUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getCatalog
#getAnnexUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getAnnex
#getSortSubUrl: http://api.weepal.cn/EAMS-XAPI/api/v1/sztd/getSortSub
getSortTreeUrl: http://127.0.0.1:8083/op/wb/getSortTree
getCatalogUrl: http://127.0.0.1:8083/op/wb/getCatalog
getAnnexUrl: http://127.0.0.1:8083/op/wb/getAnnex
getSortSubUrl: http://127.0.0.1:8083/op/wb/getSortSub