- 優化取得股票列表 API
app.get('/stocks', async function (req, res) { const stockListUpdatedString = await redisClient.get('STOCK_LIST_UPDATED'); let stocks:any; if(isEmpty(stockListUpdatedString) || dayjs().isAfter(stockListUpdatedString, 'M')) { const {data: stockIdsText} = await axios.get(STOCK_IDS_URL); const stockIdsParsed = acorn.parse( stockIdsText, { ecmaVersion: 2020 } ); // TODO: https://github.com/acornjs/acorn/issues/741 // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const stocksIdWithNameObject: {value: string}[] = stockIdsParsed.body[0].declarations[0].init.elements; const stocksIdWithName = map(stocksIdWithNameObject, 'value'); stocks = without(stocksIdWithName, '0000 加權指數', '0001 櫃買指數'); await redisClient.sAdd(STOCKS, stocks); await redisClient.set('STOCK_LIST_UPDATED', dayjs().toString()); } if(isEmpty(stocks)) stocks = await redisClient.sMembers(STOCKS); res.send(stocks); });
增加股票殖利率函式爬蟲
import {isEmpty, isUndefined, map, without} from 'lodash';
const COLLECTION = 'stock';
const STOCKS = 'STOCKS';
const STOCK_LIST_UPDATED = 'STOCK_LIST_UPDATED';
const STOCK_IDS_URL = 'https://goodinfo.tw/tw/StockLib/js/TW_STOCK_ID_NM_LIST.js?0';
const DIVIDEND_PREFIX_URL = 'https://goodinfo.tw/tw/StockDividendPolicy.asp?STOCK_ID=';
enum DividendState {
SUCCESS,
FAILURE,
NOTHING
}
interface Dividend {
[key: string]: DividendState[]
}
(async ()=> {
const mongodbClient = await mongodbConnect();
const redisClient = await redisConnect();
const app = express();
app.use(cors({credentials:true}));
app.get('/stock', async function (req, res) {
const { search } = req.query as {search:string};
if(isEmpty(search)) res.send(404);
let stock = await mongodbClient.collection(COLLECTION).findOne({
$or:[{id:search},{name:search}]
});
if(isEmpty(stock)) return res.sendStatus(404);
const {id, name, successRate, updated} = stock;
const a = dayjs().isAfter(updated, 'M');
if(isUndefined(successRate) || dayjs().isAfter(updated, 'M')) {
// TODO: https://github.com/acornjs/acorn/issues/741
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const {data: dividendText} = await axios.get(DIVIDEND_PREFIX_URL + id);
const $ = cheerio.load(dividendText);
const price = parseFloat($('body > table:nth-child(8) > tbody > tr > td:nth-child(3) > table:nth-child(1) > tbody > tr > td:nth-child(1) > table > tbody > tr:nth-child(3) > td:nth-child(1)').text());
const allAvgCashYields = parseFloat($('#divDividendSumInfo > div > div > table > tbody > tr:nth-child(4) > td:nth-child(5)').text());
const allAvgRetroactiveYields = parseFloat($('#divDividendSumInfo > div > div > table > tbody > tr:nth-child(6) > td:nth-child(5)').text());
if (isNaN(price) || allAvgRetroactiveYields === 0 || isNaN(allAvgRetroactiveYields)) {
res.sendStatus(404);
}
let yearText:string;
let year:number;
const $trs = $('#tblDetail > tbody > tr');
const dividends: Dividend = {};
for(let i = 4; i < $trs.length - 1; i++) {
let dividendState: DividendState = DividendState.NOTHING;
const $dividendTr = $trs.eq(i);
yearText = $dividendTr.children('td').eq(0).text();
if(!isNaN(yearText as never)) {
year = parseInt(yearText);
dividends[year] = [];
} else if(yearText !== '∟') continue;
const cashDividendText = $dividendTr.children('td').eq(3).text();
const stockDividendText = $dividendTr.children('td').eq(6).text();
const cashDividendSpendDaysText = $dividendTr.children('td').eq(10).text();
const stockDividendSpendDaysText = $dividendTr.children('td').eq(11).text();
if(cashDividendText !== '0' && cashDividendText !== '-') {
if(cashDividendSpendDaysText !== '-') dividendState = DividendState.SUCCESS;
else dividendState = DividendState.FAILURE;
}
if(stockDividendText !== '0' && stockDividendText !== '-' && dividendState !== DividendState.FAILURE) {
if(stockDividendSpendDaysText !== '-') dividendState = DividendState.SUCCESS;
else dividendState = DividendState.FAILURE;
}
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
dividends[year!].push(dividendState);
}
const dividendsValues = R.values(dividends);
const dividendsYears = R.keys(dividends);
const amountOfDividend = dividendsValues.length;
if(amountOfDividend === 0) {
res.sendStatus(404);
}
const dividendsFailureObject = R.filter(value => {
if(value.length === 1) {
if(value[0] === DividendState.FAILURE) return true;
else return false;
}
return R.any(R.equals(1))( R.splitAt(1, value)[1]);
}, dividends);
const dividendsFailures = R.keys(dividendsFailureObject);
const amountOfSuccess = amountOfDividend - dividendsFailures.length;
const successRate = (amountOfSuccess / amountOfDividend) * 100.00;
const dividendYearStart = dividendsYears[0];
const dividendYearEnd = dividendsYears[amountOfDividend - 1];
const updated = await mongodbClient.collection(COLLECTION).updateOne({
id
}, {
$set: {
name,
successRate,
allAvgCashYields,
allAvgRetroactiveYields,
amountOfDividend,
amountOfSuccess,
dividendYearStart,
dividendYearEnd
},
$currentDate: { updated: true },
}, {
upsert: true,
});
stock = {...stock,
successRate,
allAvgCashYields,
allAvgRetroactiveYields,
amountOfDividend,
amountOfSuccess,
dividendYearStart,
dividendYearEnd};
}
res.send(stock);
});
app.get('/stocks', async function (req, res) {
const stockListUpdatedString = await redisClient.get(STOCK_LIST_UPDATED);
let stocks:any;
if(isEmpty(stockListUpdatedString) || dayjs().isAfter(stockListUpdatedString, 'M')) {
const {data: stockIdsText} = await axios.get(STOCK_IDS_URL);
const stockIdsParsed = acorn.parse(
stockIdsText,
{ ecmaVersion: 2020 }
);
// TODO: https://github.com/acornjs/acorn/issues/741
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const stocksIdWithNameObject: {value: string}[] = stockIdsParsed.body[0].declarations[0].init.elements;
const stocksIdWithName = map(stocksIdWithNameObject, 'value');
stocks = without(stocksIdWithName, '0000 加權指數', '0001 櫃買指數');
const stockUpsertIdAndNameFunction = async (stockIdWithName: string) => {
const [id, name] = stockIdWithName.split(' ');
const updated = await mongodbClient.collection(COLLECTION).updateOne({
id
}, {
$set: {
name,
},
$currentDate: { updated: true },
}, {
upsert: true,
});
};
Promise.all(map(stocks, stockUpsertIdAndNameFunction));
await redisClient.sAdd(STOCKS, stocks);
await redisClient.set(STOCK_LIST_UPDATED, dayjs().toString());
}
if(isEmpty(stocks)) stocks = await redisClient.sMembers(STOCKS);
res.send(stocks);
});
app.listen(3000);
})();
- 增加返回錯誤
return res.sendStatus(404);