【API设计与开发实践】第1篇 API 市场观察和变迁发现

// https://market.aliyun.com/products/56956004/

localStorage.removeItem("data"); 
localStorage.removeItem("page"); 
var h1 = setInterval(function () {
    var page = document.getElementsByClassName("next-current")[0].innerText;
    var pageObj = JSON.parse(localStorage.getItem("page") || '{"pages":[], "pageCount": 0}');
    console.log("page", pageObj);
    if (pageObj.pages.indexOf(page) == -1) {
        pageObj.pages.push(page);
        pageObj.pageCount = pageObj.pages.length;
        localStorage.setItem("page", JSON.stringify(pageObj));
        localStorage.setItem("data", JSON.stringify([...document.getElementsByClassName("market-commodity-card-info")].reduce(function (obj, el) {
            var name = el.getElementsByTagName("h1")[0].getAttribute("title");
            var shopName = el.getElementsByClassName("market-commodity-card-info-row-shop-supplier")[0].getElementsByTagName("a")[0].innerText;
            console.log(name, shopName);
            if (!obj.apiMap[shopName]) {
                obj.apiMap[shopName] = [];
                obj.companyCount++;
            }
            obj.apiMap[shopName].push(name);            
            obj.apiCount++;
            if(obj.apiNames.indexOf(name) == -1) {
                obj.apiNames.push(name);
            }
            obj.uniqueApiCount = obj.apiNames.length;
            return obj;
        }, JSON.parse(localStorage.getItem("data") || '{"companyCount":0,"apiCount":0,"uniqueApiCount":0,"apiMap":{},"apiNames":[]}'))))
    }; document.getElementsByClassName("next-next")[0].click();
}, 400)

var d = JSON.parse(localStorage.getItem("data"));
console.log("data", d)
var cl = Object.keys(d.apiMap).map(m=>d.apiMap[m].length).sort()
console.log("count >= 10", cl.filter(m=>m>=10).length)
console.log("count array", cl.join(","))

// https://market.cloud.tencent.com/categories/1006

localStorage.removeItem("data"); 
localStorage.removeItem("page"); 
var h1 = setInterval(function () {
    var curPageEle = document.getElementsByClassName("mp-pagination__btn--actived")[0]
    if(!curPageEle) {
        console.log("missing curPageObj");
    }
    var page = curPageEle.innerText;
    var pageObj = JSON.parse(localStorage.getItem("page") || '{"pages":[], "pageCount": 0}');
    console.log("page", pageObj);
    if (pageObj.pages.indexOf(page) == -1) {
        pageObj.pages.push(page);
        pageObj.pageCount = pageObj.pages.length;
        localStorage.setItem("page", JSON.stringify(pageObj));
        localStorage.setItem("data", JSON.stringify([...document.getElementsByClassName("mp-synopticCard")].reduce(function (obj, el) {
            var nameEle = el.getElementsByClassName("mp-synopticCard-block-title")[0];
            if(!nameEle) {
                console.log("missing nameEle")
            }
            var name = nameEle.innerText;
            var shopNameEle = el.getElementsByClassName("deliver-company-name")[0];
            if(!shopNameEle) {
                console.log("missing shopNameEle")
            }
            var shopName = shopNameEle.innerText;
            console.log(name, shopName);
            if (!obj.apiMap[shopName]) {
                obj.apiMap[shopName] = [];
                obj.companyCount++;
            }
            obj.apiMap[shopName].push(name);            
            obj.apiCount++;
            if(obj.apiNames.indexOf(name) == -1) {
                obj.apiNames.push(name);
            }
            obj.uniqueApiCount = obj.apiNames.length;
            return obj;
        }, JSON.parse(localStorage.getItem("data") || '{"companyCount":0,"apiCount":0,"uniqueApiCount":0,"apiMap":{},"apiNames":[]}'))))
    }; document.getElementsByClassName("mp-pagination__btn--prev")[2].click();
}, 1000)

fetch("https://apis.baidu.com/api/v3/product/list?pageNo=1&pageSize=1000&cid=130&keyword=&priceFrom=0&order=desc&orderBy=weight").then(data=>data.json()).then(data=> {
    var d = data.result.reduce(function (obj, el) {
        var name = el.title;
        var shopName = el.vendorName;
        // console.log(name, shopName);
        if (!obj.apiMap[shopName]) {
            obj.apiMap[shopName] = [];
            obj.companyCount++;
        }
        obj.apiMap[shopName].push(name);            
        obj.apiCount++;
        if(obj.apiNames.indexOf(name) == -1) {
            obj.apiNames.push(name);
        }
        obj.uniqueApiCount = obj.apiNames.length;
        return obj;
    }, {"companyCount":0,"apiCount":0,"uniqueApiCount":0,"apiMap":{},"apiNames":[]});
    console.log("summary", d);
    var cl = Object.keys(d.apiMap).map(m=>d.apiMap[m].length).sort()
    console.log("count >= 10", cl.filter(m=>m>=10).length)
    console.log("count array", cl.join(","))    
})

export interface Summary {
    companyCount: number,
    apiCount: number,
    uniqueApiCount: number,
    apiMap: {[index:string]: string[] },
    apiNames: string[],
    apiCounts: number[],
    apiScaleCount: number,
    tagMap: {[index:string]: number},
    companyNames: string[]
}

const url = "https://portal.huaweicloud.com/portalsearchqueryservice/marketplacesearch?pageNo=${pageNo}&pageSize=50&typeCode=-1&form=-1&cnotallow=7&priceRange=-1&supportOS=-1&productType=-1&tagIds=&priceStart=-1&priceEnd=-1&q="

const get = (pageNumber: number): any => fetch(url.replace("${pageNo}", pageNumber)).then(data => data.json())

const main = async () => {
    console.log(`fetch the first page of Huawei`)
    const items: any[] = []
    await get(1).then(async (data: any) => {
        const pageCount = data.pagination.pageCount
        items.push(...data.pagination.items)
        for (let i = 2; i <= pageCount; i++) {
            console.log(`fetch ${i}/${pageCount}`)
            await get(i).then((d: any) => items.push(...d.pagination.items))
        }
        Deno.writeTextFileSync("data/huawei-data.json", JSON.stringify(items))
        console.log("fetch all files finished")
    })

    const summary: Summary = {
        companyCount: 0,
        apiCount: 0,
        uniqueApiCount: 0,
        apiMap: {},
        apiNames: [],
        apiCounts: [],
        apiScaleCount: 0,
        tagMap: {},
        companyNames: []
    }
    items.forEach((item: any) => {
        if (summary.apiMap[item.corporationname] === undefined){
            summary.companyNames.push(item.corporationname)
            summary.apiMap[item.corporationname] = []
            summary.companyCount++
        }
        summary.apiMap[item.corporationname].push(item.title)
        summary.apiCount++
        (item.tagnames || "").split(",").forEach((tag:string)=> {
            if(summary.tagMap[tag] == undefined) {
                summary.tagMap[tag] = 1
            } else {
                summary.tagMap[tag]++
            }
        })
    })
    const titles: string[] = []
    Object.keys(summary.apiMap).forEach((company: string) => {
        summary.apiMap[company] = summary.apiMap[company].sort()
        titles.push(...summary.apiMap[company])
    })
    summary.apiNames = titles.filter((v, i, a) => a.indexOf(v) === i).sort()
    summary.uniqueApiCount = summary.apiNames.length
    summary.apiCounts = Object.keys(summary.apiMap).map(m => summary.apiMap[m].length).sort()
    summary.apiScaleCount = summary.apiCounts.filter(m => m >= 10).length

    Deno.writeTextFileSync("data/huawei-summary.json", JSON.stringify(summary))
}

main().then(() => console.log("count finished")