mirror of
https://github.com/hempyhemp/hh-auto-reply.git
synced 2026-07-24 21:04:58 +00:00
🆕 feat(src/hh/scraper): добавляет кэширование вакансий для улучшения производительности поиска
All checks were successful
Deploy / deploy (push) Successful in 45s
All checks were successful
Deploy / deploy (push) Successful in 45s
♻️ chore(src/hh/handlers/region): очищает кэш вакансий при изменении региона ♻️ chore(src/hh/bot-commands): очищает кэш вакансий при сбросе запроса ♻️ chore(src/hh/handlers/settings): очищает кэш вакансий при изменении режима поиска ♻️ chore(src/hh/handlers/resume): очищает кэш вакансий при сохранении резюме
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import bot from '@bot'
|
import bot from '@bot'
|
||||||
import prisma from '@prisma'
|
import prisma from '@prisma'
|
||||||
import { debugFunc } from '@/hh/handlers/debug'
|
import { debugFunc } from '@/hh/handlers/debug'
|
||||||
|
import { clearVacancyCache } from './scraper.js'
|
||||||
import { handleApply } from './handlers/apply.js'
|
import { handleApply } from './handlers/apply.js'
|
||||||
import { doLogin, doLoginByPhone, handleLogin, handleLoginByEmail, handleLoginByPhone } from './handlers/auth.js'
|
import { doLogin, doLoginByPhone, handleLogin, handleLoginByEmail, handleLoginByPhone } from './handlers/auth.js'
|
||||||
import { handleSkipped, handleStatus } from './handlers/info.js'
|
import { handleSkipped, handleStatus } from './handlers/info.js'
|
||||||
@@ -91,6 +92,7 @@ const CALLBACK_HANDLERS: Record<string, CallbackHandler> = {
|
|||||||
state.queryPromptMessageId = null
|
state.queryPromptMessageId = null
|
||||||
await bot.deleteMessage(chatId, messageId).catch(() => {})
|
await bot.deleteMessage(chatId, messageId).catch(() => {})
|
||||||
await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: '' } })
|
await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: '' } })
|
||||||
|
clearVacancyCache(chatId)
|
||||||
await bot.sendMessage(chatId, '🗑 Запрос очищен')
|
await bot.sendMessage(chatId, '🗑 Запрос очищен')
|
||||||
},
|
},
|
||||||
hh_keep_max: async (chatId, messageId) => {
|
hh_keep_max: async (chatId, messageId) => {
|
||||||
@@ -274,6 +276,7 @@ export function registerHHCommands() {
|
|||||||
}
|
}
|
||||||
await bot.deleteMessage(chatId, msg.message_id).catch(() => {})
|
await bot.deleteMessage(chatId, msg.message_id).catch(() => {})
|
||||||
const updated = await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: msg.text } })
|
const updated = await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: msg.text } })
|
||||||
|
clearVacancyCache(chatId)
|
||||||
await bot.sendMessage(chatId, `✅ Запрос: «${updated.searchQuery}»`)
|
await bot.sendMessage(chatId, `✅ Запрос: «${updated.searchQuery}»`)
|
||||||
await showResumeModeStep(chatId)
|
await showResumeModeStep(chatId)
|
||||||
return
|
return
|
||||||
@@ -326,6 +329,7 @@ export function registerHHCommands() {
|
|||||||
state.queryPromptMessageId = null
|
state.queryPromptMessageId = null
|
||||||
}
|
}
|
||||||
const updated = await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: msg.text } })
|
const updated = await prisma.settings.update({ where: { telegramId: chatId }, data: { searchQuery: msg.text } })
|
||||||
|
clearVacancyCache(chatId)
|
||||||
await bot.sendMessage(chatId, `✅ Запрос: "${updated.searchQuery}"`)
|
await bot.sendMessage(chatId, `✅ Запрос: "${updated.searchQuery}"`)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import bot from '@bot'
|
import bot from '@bot'
|
||||||
import prisma from '@prisma'
|
import prisma from '@prisma'
|
||||||
|
import { clearVacancyCache } from '../scraper.js'
|
||||||
import { HH_AREAS, areaLabel } from '../areas.js'
|
import { HH_AREAS, areaLabel } from '../areas.js'
|
||||||
|
|
||||||
export async function handleRegion(chatId: number): Promise<void> {
|
export async function handleRegion(chatId: number): Promise<void> {
|
||||||
@@ -21,6 +22,7 @@ export async function handleRegion(chatId: number): Promise<void> {
|
|||||||
export async function handleAreaPick(chatId: number, messageId: number, rawCode: string): Promise<void> {
|
export async function handleAreaPick(chatId: number, messageId: number, rawCode: string): Promise<void> {
|
||||||
const code = rawCode === 'world' ? null : rawCode
|
const code = rawCode === 'world' ? null : rawCode
|
||||||
await prisma.settings.update({ where: { telegramId: chatId }, data: { area: code } })
|
await prisma.settings.update({ where: { telegramId: chatId }, data: { area: code } })
|
||||||
|
clearVacancyCache(chatId)
|
||||||
await bot.deleteMessage(chatId, messageId).catch(() => {})
|
await bot.deleteMessage(chatId, messageId).catch(() => {})
|
||||||
await bot.sendMessage(chatId, `✅ Регион: <b>${areaLabel(code)}</b>`, { parse_mode: 'HTML' })
|
await bot.sendMessage(chatId, `✅ Регион: <b>${areaLabel(code)}</b>`, { parse_mode: 'HTML' })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import bot from '@bot'
|
import bot from '@bot'
|
||||||
import prisma from '@prisma'
|
import prisma from '@prisma'
|
||||||
import { createLogger } from '@/logger'
|
import { createLogger } from '@/logger'
|
||||||
import { listResumes, NoResumeError, saveResume } from '../scraper.js'
|
import { clearVacancyCache, listResumes, NoResumeError, saveResume } from '../scraper.js'
|
||||||
import { getState } from '../state.js'
|
import { getState } from '../state.js'
|
||||||
import { escapeHtml, NO_RESUME_MARKUP, safeEdit } from '../ui.js'
|
import { escapeHtml, NO_RESUME_MARKUP, safeEdit } from '../ui.js'
|
||||||
import { startOnboarding } from './onboarding.js'
|
import { startOnboarding } from './onboarding.js'
|
||||||
@@ -94,6 +94,7 @@ export async function handleResumePick(chatId: number, messageId: number, idx: n
|
|||||||
reply_markup: { inline_keyboard: [] },
|
reply_markup: { inline_keyboard: [] },
|
||||||
})
|
})
|
||||||
await saveResume(chatId, resume)
|
await saveResume(chatId, resume)
|
||||||
|
clearVacancyCache(chatId)
|
||||||
state.pendingResumes = []
|
state.pendingResumes = []
|
||||||
await safeEdit(`✅ Резюме выбрано: ${resume.title}`, {
|
await safeEdit(`✅ Резюме выбрано: ${resume.title}`, {
|
||||||
chat_id: chatId,
|
chat_id: chatId,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import bot from '@bot'
|
import bot from '@bot'
|
||||||
import prisma from '@prisma'
|
import prisma from '@prisma'
|
||||||
import cron from 'node-cron'
|
import cron from 'node-cron'
|
||||||
|
import { clearVacancyCache } from '../scraper.js'
|
||||||
import { buildSettingsKeyboard, escapeHtml } from '../ui.js'
|
import { buildSettingsKeyboard, escapeHtml } from '../ui.js'
|
||||||
import { getState } from '../state.js'
|
import { getState } from '../state.js'
|
||||||
|
|
||||||
@@ -9,6 +10,7 @@ export async function handleSearchModeToggle(chatId: number): Promise<void> {
|
|||||||
const current = settings?.searchMode ?? 'text'
|
const current = settings?.searchMode ?? 'text'
|
||||||
const next = current === 'text' ? 'resume' : 'text'
|
const next = current === 'text' ? 'resume' : 'text'
|
||||||
await prisma.settings.update({ where: { telegramId: chatId }, data: { searchMode: next } })
|
await prisma.settings.update({ where: { telegramId: chatId }, data: { searchMode: next } })
|
||||||
|
clearVacancyCache(chatId)
|
||||||
|
|
||||||
const text = next === 'resume'
|
const text = next === 'resume'
|
||||||
? `✅ <b>Поиск по резюме включён</b>\n\n• Есть ключевые слова → <code>?text=...&resume=...</code>\n• Ключевые слова пусты → только <code>?resume=...</code>`
|
? `✅ <b>Поиск по резюме включён</b>\n\n• Есть ключевые слова → <code>?text=...&resume=...</code>\n• Ключевые слова пусты → только <code>?resume=...</code>`
|
||||||
|
|||||||
@@ -11,6 +11,28 @@ import { createStatusReporter, escapeHtml } from './ui.js'
|
|||||||
|
|
||||||
const log = createLogger('scraper')
|
const log = createLogger('scraper')
|
||||||
|
|
||||||
|
const VACANCY_CACHE_TTL = 60 * 60 * 1000
|
||||||
|
|
||||||
|
interface VacancyCacheEntry {
|
||||||
|
vacancies: Array<{ href: string, title: string }>
|
||||||
|
timer: ReturnType<typeof setTimeout>
|
||||||
|
}
|
||||||
|
|
||||||
|
const vacancyCache = new Map<string, VacancyCacheEntry>()
|
||||||
|
|
||||||
|
function vacancyCacheKey(chatId: number, query: string, area: string | undefined, searchMode: string, resumeId: string | undefined): string {
|
||||||
|
return `${chatId}:${query}:${area ?? ''}:${searchMode}:${resumeId ?? ''}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearVacancyCache(chatId: number): void {
|
||||||
|
for (const [key, entry] of vacancyCache.entries()) {
|
||||||
|
if (key.startsWith(`${chatId}:`)) {
|
||||||
|
clearTimeout(entry.timer)
|
||||||
|
vacancyCache.delete(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class NoResumeError extends Error {
|
export class NoResumeError extends Error {
|
||||||
constructor() {
|
constructor() {
|
||||||
super('no_resume')
|
super('no_resume')
|
||||||
@@ -320,33 +342,47 @@ export async function applyToJobs(
|
|||||||
|
|
||||||
await status('✅ Авторизация выполнена')
|
await status('✅ Авторизация выполнена')
|
||||||
|
|
||||||
const vacancies = await collectPageVacancies(page)
|
const cacheKey = vacancyCacheKey(chatId, query, area, searchMode, resumeId)
|
||||||
|
const vacancies = vacancyCache.get(cacheKey)?.vacancies ?? null
|
||||||
|
|
||||||
const pagerBlock = await page.$('[data-qa="pager-block"]')
|
let allVacancies: Array<{ href: string, title: string }>
|
||||||
if (pagerBlock) {
|
|
||||||
const maxPage = await page.$$eval(
|
if (vacancies) {
|
||||||
'[data-qa="pager-block"] [data-qa="pager-page"]',
|
log.info('Vacancies from cache:', vacancies.length)
|
||||||
links => Math.max(...links.map((a) => {
|
allVacancies = vacancies
|
||||||
const pageParam = new URL((a as HTMLAnchorElement).href).searchParams.get('page')
|
await keep(`✅ Вакансий из кэша: ${allVacancies.length}`)
|
||||||
return Number(pageParam ?? 0)
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
log.divider('CollectPageVacancies')
|
|
||||||
log.info('URL:', page.url())
|
|
||||||
log.info('Max page:', maxPage)
|
|
||||||
for (let p = 1; p <= maxPage; p++) {
|
|
||||||
const pageUrl = buildVacancySearchUrl(query, searchMode, resumeId, area, p)
|
|
||||||
await page.goto(pageUrl, { waitUntil: 'domcontentloaded' })
|
|
||||||
await page.waitForSelector('[data-qa="vacancy-serp__vacancy"]', { timeout: 10000 }).catch(() => null)
|
|
||||||
const more = await collectPageVacancies(page)
|
|
||||||
log.divider('Page Vacancies')
|
|
||||||
log.info(more)
|
|
||||||
vacancies.push(...more)
|
|
||||||
await status(`🔎 Страница ${p}`)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
else {
|
||||||
|
allVacancies = await collectPageVacancies(page)
|
||||||
|
|
||||||
await keep(`✅ Вакансий найдено: ${vacancies.length}`)
|
const pagerBlock = await page.$('[data-qa="pager-block"]')
|
||||||
|
if (pagerBlock) {
|
||||||
|
const maxPage = await page.$$eval(
|
||||||
|
'[data-qa="pager-block"] [data-qa="pager-page"]',
|
||||||
|
links => Math.max(...links.map((a) => {
|
||||||
|
const pageParam = new URL((a as HTMLAnchorElement).href).searchParams.get('page')
|
||||||
|
return Number(pageParam ?? 0)
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
log.divider('CollectPageVacancies')
|
||||||
|
log.info('URL:', page.url())
|
||||||
|
log.info('Max page:', maxPage)
|
||||||
|
for (let p = 1; p <= maxPage; p++) {
|
||||||
|
const pageUrl = buildVacancySearchUrl(query, searchMode, resumeId, area, p)
|
||||||
|
await page.goto(pageUrl, { waitUntil: 'domcontentloaded' })
|
||||||
|
await page.waitForSelector('[data-qa="vacancy-serp__vacancy"]', { timeout: 10000 }).catch(() => null)
|
||||||
|
const more = await collectPageVacancies(page)
|
||||||
|
log.divider('Page Vacancies')
|
||||||
|
log.info(more)
|
||||||
|
allVacancies.push(...more)
|
||||||
|
await status(`🔎 Страница ${p}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const timer = setTimeout(() => vacancyCache.delete(cacheKey), VACANCY_CACHE_TTL).unref()
|
||||||
|
vacancyCache.set(cacheKey, { vacancies: allVacancies, timer })
|
||||||
|
await keep(`✅ Вакансий найдено: ${allVacancies.length}`)
|
||||||
|
}
|
||||||
|
|
||||||
const resumes = await prisma.resume.findMany({ where: { telegramId: chatId } })
|
const resumes = await prisma.resume.findMany({ where: { telegramId: chatId } })
|
||||||
const settings = await prisma.settings.findUnique({ where: { telegramId: chatId } })
|
const settings = await prisma.settings.findUnique({ where: { telegramId: chatId } })
|
||||||
@@ -366,7 +402,7 @@ export async function applyToJobs(
|
|||||||
let appliedCount = 0
|
let appliedCount = 0
|
||||||
let consecutiveSkips = 0
|
let consecutiveSkips = 0
|
||||||
const MAX_CONSECUTIVE_SKIPS = 20
|
const MAX_CONSECUTIVE_SKIPS = 20
|
||||||
for (const vacancy of vacancies) {
|
for (const vacancy of allVacancies) {
|
||||||
if (appliedCount >= maxApplies)
|
if (appliedCount >= maxApplies)
|
||||||
break
|
break
|
||||||
if (consecutiveSkips >= MAX_CONSECUTIVE_SKIPS) {
|
if (consecutiveSkips >= MAX_CONSECUTIVE_SKIPS) {
|
||||||
|
|||||||
Reference in New Issue
Block a user