Rehber TDK Sözlük'ten Türkçe kelime listesi elde etme

import requests
r = requests.get("https://sozluk.gov.tr/autocomplete.json")

ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

requests.get("https://sozluk.gov.tr/autocomplete.json", headers={"User-Agent": "pythonyilannnn"})

r = requests.get("https://sozluk.gov.tr/autocomplete.json", headers={"User-Agent": "yilannntisss"})

r.json()

[{'madde': '-den yana'},
 {'madde': '-den yana çıkmak'},
 {'madde': '-den yana olmak'},
 {'madde': "-di'li geçmiş zaman"},
 {'madde': "-miş'li geçmiş zaman"},
 {'madde': '...-e gelince'},
 {'madde': '... -e kuvvet'},
 {'madde': '...-inde değil'},
 {'madde': '...-in karesi'},
 {'madde': '...-meye görsün (veya gör)'},
 ...
]

autocompletions = [girdi["madde"] for girdi in r.json()]

['-den yana',
 '-den yana çıkmak',
 '-den yana olmak',
 "-di'li geçmiş zaman",
 "-miş'li geçmiş zaman",
 '...-e gelince',
 '... -e kuvvet',
 '...-inde değil',
 '...-in karesi',
 '...-meye görsün (veya gör)',
 ...
]

word_list = [ac for ac in autocompletions if len(ac) and all(letter.isalpha() and letter.islower() for letter in ac)]

['a',
 'ab',
 'aba',
 'aba',
 'abacı',
 'abacılık',
 'abadi',
 'abajur',
 'abajurcu',
 'abajurculuk',
 'abajurlu',
 'abajursuz',
 'abaküs',
 ...
]

word_list = list(set(word_list))

SPECIAL_LETTERS_MAP = {"â": "a", "û": "u", "î": "i"}

for special_letter, hat_removed in SPECIAL_LETTERS_MAP.items():
    word_list = [word.replace(special_letter, hat_removed) for word in word_list]

word_list = list(set(word_list))

import json
import os
import sys
from typing import List

import requests
from pydantic import BaseModel, RootModel

""" JSON formati
[
    ...
    {"madde": "ala"},
    {"madde": "geyik"},
    ...
]
"""


class Autocompletion(BaseModel):
    madde: str


class TdkData(RootModel):
    root: List[Autocompletion]


# fmt: off
# Normalde alfabede ı, i'den sonra geliyor.
LOWERCASE_LETTERS = ["a", "b", "c", "ç", "d", "e", "f", "g", "ğ", "h", "ı", "i", "j", "k", "l", "m", "n", "o", "ö", "p", "r", "s", "ş", "t", "u", "ü", "v", "y", "z"]
SPECIAL_LETTERS_MAP = {"â": "a", "û": "u", "î": "i"}
# fmt: on


def fetch_autocompletions() -> List[str]:
    req = requests.get(
        "https://sozluk.gov.tr/autocomplete.json",
        headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
        },
    )
    autocompletions = TdkData.model_validate(req.json()).root
    autocompletions = [ac.madde for ac in autocompletions]
    return autocompletions


def get_word_list(autocompletions: List[str]) -> List[str]:
    """
    Kelime olma sartlari:
    - Bos olmayacak
    - Sadece harflerden olusacak
    - Kucuk harflerden olusacak (Ozel kelimeleri haric tutmak icin)
    """
    is_word = lambda expr: len(expr) > 0 and all(
        char.isalpha() and char.islower() for char in expr
    )
    word_list = [ac for ac in autocompletions if is_word(ac)]

    # Ozel harfleri halledelim
    for i in range(len(word_list)):
        for original_letter, new_letter in SPECIAL_LETTERS_MAP.items():
            word_list[i] = word_list[i].replace(original_letter, new_letter)

    # Onceki islemden sonra "ala" gibi kelimeler birden fazla kez gececek
    word_list = list(set(word_list))

    # Kelimeleri siralayalim
    word_list.sort(key=lambda word: list(map(LOWERCASE_LETTERS.index, word)))

    return word_list


if __name__ == "__main__":
    autocompletions = fetch_autocompletions()
    words = get_word_list(autocompletions)

    with open(os.path.join(sys.path[0], "words.json"), "w", encoding="utf-8") as f:
        json.dump(words, f, indent=2, ensure_ascii=False)

import fs from "node:fs/promises";
import path, { dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { z } from "zod";

const __dirname = dirname(fileURLToPath(import.meta.url));

async function fetchAutocompletions(): Promise<string[]> {
  let data: unknown;
  try {
    data = await (
      await fetch("https://sozluk.gov.tr/autocomplete.json")
    ).json();
  } catch (e) {
    console.log("Veriyi cekerken bir hata meydana geldi:");
    throw e;
  }

  // {madde: <kelime>} listesi
  const autocompletionsSchema = z
    .object({
      madde: z.string(),
    })
    .array();

  // Format kontrolu
  const parseResult = autocompletionsSchema.safeParse(data);

  if (parseResult.error) {
    throw new Error("Beklenmedik veri formatiyla karsilasildi");
  }

  const autocompletions = parseResult.data.map(({ madde }) => madde);
  autocompletions.sort(Intl.Collator("tr").compare); // Turkce alfabesine gore sirala

  // Alfabede normalde i, ı'dan once geliyor ama JS'te durum tam tersi...

  return autocompletions;
}

function extractWords(autocompletions: string[]): string[] {
  /*
    Tum harfleri kucuk olan Turkce (Unicode) kelime kontrolu.

    \p{Ll} -> Lowercase Unicode harf
    +      -> En az bir harf
    ^...$  -> Bastan sona kadar
    u      -> Unicode destegi etkinlestirme modifier'i
  */
  function isLowerCaseWord(str: string): boolean {
    return /^\p{Ll}+$/u.test(str);
  }

  const words = autocompletions.filter((ac) => isLowerCaseWord(ac));

  const specialLetterMapping = { â: "a", û: "u", î: "i" };
  const specialLetters = Object.keys(specialLetterMapping);

  const words_specialLettersReplaced = words.map((word) =>
    word.replace(new RegExp(specialLetters.join("|"), "gu"), (m) => {
      return specialLetterMapping[m];
    }),
  );

  const uniqueWords = [...new Set(words_specialLettersReplaced)];

  return uniqueWords;
}

const autocompletions = await fetchAutocompletions();
const words = extractWords(autocompletions);

await fs.writeFile(
  path.join(__dirname, "words.json"),
  JSON.stringify(words, null, 2),
);