hts/apps/blogai/lib/mdx-helper.ts

import { promises as fs } from "fs";
import path from "path";
import GithubSlugger from "github-slugger";
import { type MDXRemoteSerializeResult } from "next-mdx-remote";
import { serialize } from "next-mdx-remote/serialize";
import { notFound } from "next/navigation";
import rehypeAutolinkHeadings from "rehype-autolink-headings";
import rehypeCodeTitles from "rehype-code-titles";
import rehypePrettyCode from "rehype-pretty-code";
import rehypeSlug from "rehype-slug";
import remarkGfm from "remark-gfm";
import { BUNDLED_LANGUAGES, type HighlighterOptions, getHighlighter } from "shiki";
import gitHubLight from "shiki/themes/github-light.json";
import service from "./http/service";
import { ArticleData, QAData } from "@/components/article/article";

export const BLOG_PATH = path.join(process.cwd(), "content", "blog");
export const CHANGELOG_PATH = path.join(process.cwd(), "content", "changelog");
export const POLICY_PATH = path.join(process.cwd(), "content", "policies");
export const JOBS_PATH = path.join(process.cwd(), "content", "jobs");

export const changelogFilePaths = fs.readdir(CHANGELOG_PATH);

// console.log(BLOG_PATH)


export const getFilePaths = async (contentPath: string) => {
  return await fs.readdir(contentPath);
};
export const raw = async ({
  contentPath,
  filepath,
}: {
  contentPath: string;
  filepath: string;
}) => {

  console.log(contentPath, filepath)
  try {
    const fileContent = await fs.readFile(`${contentPath}/${filepath}.mdx`, "utf-8");


    return fileContent;
  } catch {
    return notFound();
  }
};

type Headings = {
  slug: string | undefined;
  level: string;
  text: string | undefined;

};

type Post<TFrontmatter> = {
  serialized: MDXRemoteSerializeResult;
  frontmatter: TFrontmatter;
  headings: Headings[];
};

type AIPost<TFrontmatter> = {
  serialized: MDXRemoteSerializeResult;
  frontmatter: TFrontmatter;
  headings: Headings;
};

type Changelog<TFrontmatter> = {
  serialized: MDXRemoteSerializeResult;
  frontmatter: TFrontmatter;
};

type ChangeLogFrontmatter = {
  title: string;
  date: string;
  description: string;
};

type Policy<TFrontmatter> = {
  serialized: MDXRemoteSerializeResult;
  frontmatter: TFrontmatter;
};

type PolicyFrontmatter = {
  title: string;
};

type Job<TFrontmatter> = {
  serialized: MDXRemoteSerializeResult;
  frontmatter: TFrontmatter;
};
type JobFrontmatter = {
  title: string;
  description: string;
  visible: boolean;
  salary: string;
};

type Frontmatter = {
  title: string;
  date: string;
  description: string;
  author: string;
  visible: boolean | undefined;
  salary: string | undefined;
  level: string | undefined;
};

type AIFrontmatter = {
  id:string;
  p_name:string;
  title: string;
  date: string;
  description: string;
  author: string;
  image_url: string;
  logo_url: string;
  model_parameter: number;
  visible: boolean | undefined;
  salary: string | undefined;
  level: string | undefined;
  tags?: string[];
};

const options = {
  theme: gitHubLight,
  getHighlighter: (options: HighlighterOptions) =>
    getHighlighter({
      ...options,
      langs: [...BUNDLED_LANGUAGES],
    }),
  defaultLang: {
    block: "typescript",
  },
};
// Serialize the MDX content and parse the frontmatter
export const mdxSerialized = async ({ rawMdx }: { rawMdx: string }) => {
  return await serialize(rawMdx, {
    parseFrontmatter: true,
    mdxOptions: {
      remarkPlugins: [remarkGfm],
      rehypePlugins: [
        [rehypePrettyCode, options],
        rehypeAutolinkHeadings,
        rehypeSlug,
        rehypeCodeTitles,
      ],
    },
  });
};

const getHeadings = async ({ rawMdx }: { rawMdx: string }) => {
  const slugger = new GithubSlugger();
  const regXHeader = /\n(?<flag>#{1,6})\s+(?<content>.+)/g;
  const headings = Array.from(rawMdx.matchAll(regXHeader)).map(({ groups }) => {
    const flag = groups?.flag;
    const content = groups?.content;
    return {
      level: flag?.length === 1 ? "one" : flag?.length === 2 ? "two" : "three",
      text: content,
      slug: content ? slugger.slug(content) : undefined,
    };
  });
  return headings;
};

const getMoreContent = async ({
  contentPath,
  filepath,
}: {
  contentPath: string;
  filepath: string;
}) => {
  const moreContent = await fs.readdir(contentPath);
  const moreContentFiltered = moreContent
    .filter((path) => /\.mdx?$/.test(path))
    .filter((post) => post !== filepath)
    .slice(0, 2);


  console.log("-------", moreContentFiltered)
  return moreContentFiltered;
};

export const getAllMDXData = async ({
  contentPath,
}: {
  contentPath: string;
}) => {
  const allPosts = await fs.readdir(contentPath);
  const allPostsFiltered = allPosts.filter((path) => /\.mdx?$/.test(path));
  const allPostsData = await Promise.all(
    allPostsFiltered.map(async (post) => {
      const rawMdx = await raw({
        contentPath,
        filepath: post.replace(/\.mdx?$/, ""),
      });
      const serializedMdx = await mdxSerialized({ rawMdx });
      const frontmatter = serializedMdx.frontmatter as Frontmatter;
      return {
        frontmatter,
        slug: post.replace(/\.mdx$/, ""),
      };
    }),
  );
  return allPostsData;
};

export const getContentData = async ({
  contentPath,
  filepath,
}: {
  contentPath: string;
  filepath: string;
}) => {
  const moreContent = await getMoreContent({ contentPath, filepath });
  const moreContentData = await Promise.all(
    moreContent.map(async (content) => {
      const rawMdx = await raw({
        contentPath,
        filepath: content.replace(/\.mdx?$/, ""),
      });
      const serializedMdx = await mdxSerialized({ rawMdx });
      const frontmatter = serializedMdx.frontmatter as Frontmatter;
      return {
        frontmatter,
        slug: content.replace(/\.mdx$/, ""),
      };
    }),
  );
  return moreContentData;
};

export const getPost = async (filepath: string): Promise<Post<Frontmatter>> => {
  const rawMdx = await raw({ contentPath: BLOG_PATH, filepath: filepath });

  console.log("rawMdx", rawMdx)
  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as Frontmatter;
  const headings = await getHeadings({ rawMdx });
  return {
    frontmatter,
    serialized,
    headings,
  };
};


export const getPostContent = async (language: string, slug: string): Promise<AIPost<AIFrontmatter>> => {
  // const rawMdx = await raw({ contentPath: BLOG_PATH, filepath: filepath });

  const PageSize = 1


  let data: ArticleData = await service.post('/api/v1/news/list', {
    "id": Number(slug),
    // "tag": "", // #Blockchain
    language,
    "page_no": 0,
    "page_size": PageSize
  }, {
    headers: {
      // 'Authorization': token
    }
  }).then((result: any) => {
    console.log("result:", result)

    if (result && result.header.code != 0) {

      return
    }

    return result.data.list[0]

  }).catch((err) => {
    console.log(err);

  });

  let rawMdx = data.content

  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as AIFrontmatter;
  frontmatter.id = String(data.id)
  frontmatter.p_name = data.p_name
  frontmatter.title = data.main_title
  frontmatter.description = data.sub_title || data.main_title
  frontmatter.date = data.updated_time
  frontmatter.tags = data.tags
  frontmatter.image_url = data.image_url
  frontmatter.logo_url = data.logo_url
  frontmatter.model_parameter = data.model_parameter
  // const headings = data.main_title;


  // 描述：来自sub_title字段，如果没有，则跟main_title一致
  // 关键词：来自seo_keywords字段，如果seo_keywords为空，而取keywords字段，如果都为空，则跟首页的关键字一致

  // 特殊处理
  let text = ""
  if (data.seo_keywords.length > 0 && typeof data.seo_keywords == "string") {
    try {
      const parsedArray = JSON.parse(data.seo_keywords); // 尝试解析字符串为数组
      console.log("---parsedArray---", parsedArray)
      text = Array.isArray(parsedArray) ? parsedArray.join(', ') : ""
    } catch (error) {
      console.error('解析失败：seo_keywords', error); // 输出解析失败的错误信息
    }
  } else if (Array.isArray(data.seo_keywords)) {
    text = data.seo_keywords.join(', ')
  }

  if (text.length == 0 && typeof data.keywords == "string") {
    try {
      const parsedArray = JSON.parse(data.keywords);
      text = Array.isArray(parsedArray) ? parsedArray.join(', ') : ""
    } catch (error) {
      console.error('解析失败：seo_keywords', data.seo_keywords, error);
    }
  } else if (Array.isArray(data.keywords)) {
    text = data.keywords.join(', ')
  }


  console.log('----------rawMdx:', data)
  return {
    frontmatter,
    serialized,
    headings: {
      slug: data.main_title,
      level: "",
      text: text,
    },
  };
};

export const getQAContent = async (language: string, slug: string): Promise<AIPost<AIFrontmatter>> => {
  // const rawMdx = await raw({ contentPath: BLOG_PATH, filepath: filepath });

  const PageSize = 1
  let data: QAData = await service.post('/api/v1/qa/list', {
    "id": Number(slug),
    // "tag": "", // #Blockchain
    language,
    "page_no": 0,
    "page_size": PageSize
  }, {
    headers: {
      // 'Authorization': token
    }
  }).then((result: any) => {
    console.log("result:", result)

    if (result && result.header.code != 0) {

      return
    }

    return result.data.list[0]

  }).catch((err) => {
    console.log(err);

  });

  let rawMdx = data.answer

  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as AIFrontmatter;
  frontmatter.title = data.question
  frontmatter.description = data.question
  frontmatter.date = data.updated_time
  // frontmatter.tags = data.tags
  // const headings = data.main_title;

  console.log('----------rawMdx:', data)
  return {
    frontmatter,
    serialized,
    headings: {
      slug: data.question,
      level: "",
      text: "",
    },
  };
};

export const getChangelog = async (filepath: string): Promise<Changelog<ChangeLogFrontmatter>> => {
  const rawMdx = await raw({ contentPath: CHANGELOG_PATH, filepath: filepath });
  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as ChangeLogFrontmatter;

  return {
    frontmatter,
    serialized,
  };
};

export const getPolicy = async (filepath: string): Promise<Policy<PolicyFrontmatter>> => {
  const rawMdx = await raw({ contentPath: POLICY_PATH, filepath: filepath });
  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as PolicyFrontmatter;

  return {
    frontmatter,
    serialized,
  };
};

export const getJob = async (filepath: string): Promise<Job<JobFrontmatter>> => {
  const rawMdx = await raw({ contentPath: JOBS_PATH, filepath: filepath });
  // Serialize the MDX content and parse the frontmatter
  const serialized = await mdxSerialized({ rawMdx });
  const frontmatter = serialized.frontmatter as JobFrontmatter;

  return {
    frontmatter,
    serialized,
  };
};

export const getAllJobsData = async ({
  contentPath,
}: {
  contentPath: string;
}) => {
  const allJobs = await fs.readdir(contentPath);
  const allJosFiltered = allJobs.filter((path) => /\.mdx?$/.test(path));
  const allPostsData = await Promise.all(
    allJosFiltered.map(async (post) => {
      const rawMdx = await raw({
        contentPath,
        filepath: post.replace(/\.mdx?$/, ""),
      });
      const serializedMdx = await mdxSerialized({ rawMdx });
      const frontmatter = serializedMdx.frontmatter as JobFrontmatter;
      if (frontmatter.visible === false) {
        return;
      }
      return {
        frontmatter,
        slug: post.replace(/\.mdx$/, ""),
      };
    }),
  );
  return allPostsData;
};