Add content writing command

bin/readme.md

@@ -9,12 +9,22 @@ Generates a list of all the resources links in any roadmap file.
 
 Compresses all the JSON files in the `public/jsons` folder
 
+## `update-sponsors.cjs`
+
+Updates the sponsor ads on each roadmap page with the latest sponsor information in the Excel sheet.
+
 ## `roadmap-content.cjs`
 
+Currently, for any new roadmaps that we add, we do create the interactive roadmap but we end up leaving the content empty in the roadmap till we get time to fill it up manually.
+
+This script populates all the content files with some minimal content from OpenAI so that the users visiting the website have something to read in the interactive roadmap till we get time to fill it up manually.
+
+## `roadmap-dirs.cjs`
+
 This command is used to create the content folders and files for the interactivity of the roadmap. You can use the below command to generate the roadmap skeletons inside a roadmap directory:
 
 ```bash
-npm run roadmap-content [frontend|backend|devops|...]
+npm run roadmap-dirs [frontend|backend|devops|...]
 ```
 
 For the content skeleton to be generated, we should have proper grouping, and the group names in the project files. You can follow the steps listed below in order to add the meta information to the roadmap.

bin/roadmap-content.cjs

@@ -20,6 +20,12 @@ if (!allowedRoadmapIds.includes(roadmapId)) {
 }
 
 const ROADMAP_CONTENT_DIR = path.join(ALL_ROADMAPS_DIR, roadmapId, 'content');
+const { Configuration, OpenAIApi } = require('openai');
+const configuration = new Configuration({
+  apiKey: OPEN_AI_API_KEY,
+});
+
+const openai = new OpenAIApi(configuration);
 
 function getFilesInFolder(folderPath, fileList = {}) {
   const files = fs.readdirSync(folderPath);
@@ -44,38 +50,88 @@ function getFilesInFolder(folderPath, fileList = {}) {
   return fileList;
 }
 
-const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR);
-
-const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`));
-const groups = roadmapJson?.mockup?.controls?.control?.filter(
-  (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link')
-);
+function writeTopicContent(topicTitle) {
+  console.log(`Genearting '${topicTitle}'...`);
+
+  const instruction = `Write a short paragraph explaining '${topicTitle}' in ${roadmapId}. Avoid any text similar to "In ${roadmapId}, ${topicTitle} refers to" and write it as a direct explanation of the topic.`;
+
+  return new Promise((resolve, reject) => {
+    openai
+      .createChatCompletion({
+        model: 'gpt-4',
+        messages: [
+          {
+            role: 'user',
+            content: instruction,
+          },
+        ],
+      })
+      .then((response) => {
+        const article = response.data.choices[0].message.content;
+
+        resolve(article);
+      })
+      .catch((err) => {
+        reject(err);
+      });
+  });
+}
 
-groups.forEach((group) => {
-  const topicId = group?.properties?.controlName;
-  const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties
-    ?.text;
-  const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/');
-  const contentFilePath = topicUrlToPathMapping[currTopicUrl];
+async function run() {
+  const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR);
 
-  const currentFileContent = fs.readFileSync(contentFilePath, 'utf8');
-  const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == '';
+  const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`));
+  const groups = roadmapJson?.mockup?.controls?.control?.filter(
+    (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link')
+  );
 
-  if (!isFileEmpty) {
-    console.log(`${topicId} not empty. Ignoring...`);
-    return;
+  if (!OPEN_AI_API_KEY) {
+    console.log('----------------------------------------');
+    console.log('OPEN_AI_API_KEY not found. Skipping openai api calls...');
+    console.log('----------------------------------------');
   }
 
-  let newFileContent = `# ${topicTitle}`;
+  for (let group of groups) {
+    const topicId = group?.properties?.controlName;
+    const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties
+      ?.text;
+    const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/');
+    const contentFilePath = topicUrlToPathMapping[currTopicUrl];
 
-  if (!OPEN_AI_API_KEY) {
-    console.log(`OPEN_AI_API_KEY not set. Only adding title to ${topicId}..`);
+    const currentFileContent = fs.readFileSync(contentFilePath, 'utf8');
+    const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == '';
+
+    if (!isFileEmpty) {
+      console.log(`Ignoring ${topicId}. Not empty.`);
+      continue;
+    }
+
+    let newFileContent = `# ${topicTitle}`;
+
+    if (!OPEN_AI_API_KEY) {
+      console.log(`Writing ${topicId}..`);
+      fs.writeFileSync(contentFilePath, newFileContent, 'utf8');
+      continue;
+    }
+
+    const topicContent = await writeTopicContent(topicTitle);
+    newFileContent += `\n\n${topicContent}`;
+
+    console.log(`Writing ${topicId}..`);
     fs.writeFileSync(contentFilePath, newFileContent, 'utf8');
-    return;
+
+    // console.log(currentFileContent);
+    // console.log(currTopicUrl);
+    // console.log(topicTitle);
+    // console.log(topicUrlToPathMapping[currTopicUrl]);
   }
+}
 
-  // console.log(currentFileContent);
-  // console.log(currTopicUrl);
-  // console.log(topicTitle);
-  // console.log(topicUrlToPathMapping[currTopicUrl]);
-});
+run()
+  .then(() => {
+    console.log('Done');
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

package.json

@@ -35,6 +35,7 @@
     "gh-pages": "^5.0.0",
     "js-yaml": "^4.1.0",
     "markdown-it": "^13.0.1",
+    "openai": "^3.2.1",
     "prettier": "^2.8.7",
     "prettier-plugin-astro": "^0.8.0"
   }

pnpm-lock.yaml

@@ -12,6 +12,7 @@ specifiers:
   markdown-it: ^13.0.1
   node-html-parser: ^6.1.5
   npm-check-updates: ^16.8.0
+  openai: ^3.2.1
   prettier: ^2.8.7
   prettier-plugin-astro: ^0.8.0
   rehype-external-links: ^2.0.1
@@ -35,6 +36,7 @@ devDependencies:
   gh-pages: 5.0.0
   js-yaml: 4.1.0
   markdown-it: 13.0.1
+  openai: 3.2.1
   prettier: 2.8.7
   prettier-plugin-astro: 0.8.0
 
@@ -1185,6 +1187,10 @@ packages:
     resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==}
     dev: true
 
+  /asynckit/0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+    dev: true
+
   /autoprefixer/10.4.13_postcss@8.4.21:
     resolution: {integrity: sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==}
     engines: {node: ^10 || ^12 || >=14}
@@ -1201,6 +1207,14 @@ packages:
       postcss-value-parser: 4.2.0
     dev: false
 
+  /axios/0.26.1:
+    resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==}
+    dependencies:
+      follow-redirects: 1.15.2
+    transitivePeerDependencies:
+      - debug
+    dev: true
+
   /bail/2.0.2:
     resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==}
     dev: false
@@ -1563,6 +1577,13 @@ packages:
       color-string: 1.9.1
     dev: false
 
+  /combined-stream/1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      delayed-stream: 1.0.0
+    dev: true
+
   /comma-separated-tokens/2.0.3:
     resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
     dev: false
@@ -1743,6 +1764,11 @@ packages:
   /defined/1.0.1:
     resolution: {integrity: sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==}
 
+  /delayed-stream/1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+    dev: true
+
   /delegates/1.0.0:
     resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
     dev: false
@@ -2075,11 +2101,30 @@ packages:
       pkg-dir: 4.2.0
     dev: false
 
+  /follow-redirects/1.15.2:
+    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
+    engines: {node: '>=4.0'}
+    peerDependencies:
+      debug: '*'
+    peerDependenciesMeta:
+      debug:
+        optional: true
+    dev: true
+
   /form-data-encoder/2.1.4:
     resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==}
     engines: {node: '>= 14.17'}
     dev: false
 
+  /form-data/4.0.0:
+    resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
+    engines: {node: '>= 6'}
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+    dev: true
+
   /fp-and-or/0.1.3:
     resolution: {integrity: sha512-wJaE62fLaB3jCYvY2ZHjZvmKK2iiLiiehX38rz5QZxtdN8fVPJDeZUiVvJrHStdTc+23LHlyZuSEKgFc0pxi2g==}
     engines: {node: '>=10'}
@@ -3384,6 +3429,18 @@ packages:
       braces: 3.0.2
       picomatch: 2.3.1
 
+  /mime-db/1.52.0:
+    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+    engines: {node: '>= 0.6'}
+    dev: true
+
+  /mime-types/2.1.35:
+    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+    engines: {node: '>= 0.6'}
+    dependencies:
+      mime-db: 1.52.0
+    dev: true
+
   /mime/3.0.0:
     resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
     engines: {node: '>=10.0.0'}
@@ -3804,6 +3861,15 @@ packages:
       is-docker: 2.2.1
       is-wsl: 2.2.0
 
+  /openai/3.2.1:
+    resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==}
+    dependencies:
+      axios: 0.26.1
+      form-data: 4.0.0
+    transitivePeerDependencies:
+      - debug
+    dev: true
+
   /ora/6.1.2:
     resolution: {integrity: sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}