vault backup: 2026-07-18 23:06:13
This commit is contained in:
Vendored
+3
-1
@@ -1,5 +1,7 @@
|
|||||||
[
|
[
|
||||||
"obsidian-git",
|
"obsidian-git",
|
||||||
"iconic",
|
"iconic",
|
||||||
"folder-links"
|
"folder-links",
|
||||||
|
"settings-search",
|
||||||
|
"omnisearch"
|
||||||
]
|
]
|
||||||
+23
-1
@@ -28,7 +28,29 @@
|
|||||||
},
|
},
|
||||||
"appIcons": {},
|
"appIcons": {},
|
||||||
"tabIcons": {},
|
"tabIcons": {},
|
||||||
"fileIcons": {},
|
"fileIcons": {
|
||||||
|
"00 Inbox": {
|
||||||
|
"icon": "lucide-inbox"
|
||||||
|
},
|
||||||
|
"10 Knowledge": {
|
||||||
|
"icon": "lucide-school"
|
||||||
|
},
|
||||||
|
"20 Work": {
|
||||||
|
"icon": "lucide-building"
|
||||||
|
},
|
||||||
|
"30 Personal Projects": {
|
||||||
|
"icon": "lucide-user"
|
||||||
|
},
|
||||||
|
"40 Journal": {
|
||||||
|
"icon": "lucide-pen-tool"
|
||||||
|
},
|
||||||
|
"50 Principles": {
|
||||||
|
"icon": "lucide-star"
|
||||||
|
},
|
||||||
|
"90 Archives": {
|
||||||
|
"icon": "lucide-archive"
|
||||||
|
}
|
||||||
|
},
|
||||||
"bookmarkIcons": {},
|
"bookmarkIcons": {},
|
||||||
"tagIcons": {},
|
"tagIcons": {},
|
||||||
"propertyIcons": {},
|
"propertyIcons": {},
|
||||||
|
|||||||
+38
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"biggerIcons": "mobile",
|
||||||
|
"clickableIcons": "desktop",
|
||||||
|
"showAllFileIcons": false,
|
||||||
|
"showAllFolderIcons": false,
|
||||||
|
"minimalFolderIcons": true,
|
||||||
|
"showMarkdownTabIcons": true,
|
||||||
|
"showTitleIcons": true,
|
||||||
|
"showTagPillIcons": false,
|
||||||
|
"showMenuActions": true,
|
||||||
|
"showSuggestionIcons": false,
|
||||||
|
"showQuickSwitcherIcons": true,
|
||||||
|
"showMoveFileIcons": true,
|
||||||
|
"showItemName": "desktop",
|
||||||
|
"biggerSearchResults": "mobile",
|
||||||
|
"maxSearchResults": 50,
|
||||||
|
"colorPicker1": "list",
|
||||||
|
"colorPicker2": "rgb",
|
||||||
|
"uncolorHover": false,
|
||||||
|
"uncolorDrag": false,
|
||||||
|
"uncolorSelect": false,
|
||||||
|
"uncolorQuick": false,
|
||||||
|
"maxBackups": 2,
|
||||||
|
"dialogState": {
|
||||||
|
"iconMode": true,
|
||||||
|
"emojiMode": false,
|
||||||
|
"rulePage": "file"
|
||||||
|
},
|
||||||
|
"appIcons": {},
|
||||||
|
"tabIcons": {},
|
||||||
|
"fileIcons": {},
|
||||||
|
"bookmarkIcons": {},
|
||||||
|
"tagIcons": {},
|
||||||
|
"propertyIcons": {},
|
||||||
|
"ribbonIcons": {},
|
||||||
|
"fileRules": [],
|
||||||
|
"folderRules": []
|
||||||
|
}
|
||||||
+43
@@ -0,0 +1,43 @@
|
|||||||
|
{
|
||||||
|
"useCache": true,
|
||||||
|
"hideExcluded": false,
|
||||||
|
"recencyBoost": "0",
|
||||||
|
"downrankedFoldersFilters": [],
|
||||||
|
"ignoreDiacritics": true,
|
||||||
|
"ignoreArabicDiacritics": false,
|
||||||
|
"indexedFileTypes": [],
|
||||||
|
"indexFilesWithoutExtension": false,
|
||||||
|
"displayTitle": "",
|
||||||
|
"PDFIndexing": false,
|
||||||
|
"officeIndexing": false,
|
||||||
|
"imagesIndexing": false,
|
||||||
|
"aiImageIndexing": false,
|
||||||
|
"unsupportedFilesIndexing": "default",
|
||||||
|
"splitCamelCase": false,
|
||||||
|
"openInNewPane": false,
|
||||||
|
"vimLikeNavigationShortcut": false,
|
||||||
|
"ribbonIcon": true,
|
||||||
|
"showExcerpt": true,
|
||||||
|
"maxEmbeds": 5,
|
||||||
|
"renderLineReturnInExcerpts": true,
|
||||||
|
"showCreateButton": false,
|
||||||
|
"highlight": true,
|
||||||
|
"showPreviousQueryResults": true,
|
||||||
|
"simpleSearch": false,
|
||||||
|
"tokenizeUrls": false,
|
||||||
|
"fuzziness": "1",
|
||||||
|
"weightBasename": 10,
|
||||||
|
"weightDirectory": 7,
|
||||||
|
"weightH1": 6,
|
||||||
|
"weightH2": 5,
|
||||||
|
"weightH3": 4,
|
||||||
|
"weightUnmarkedTags": 2,
|
||||||
|
"weightCustomProperties": [],
|
||||||
|
"httpApiEnabled": false,
|
||||||
|
"httpApiPort": "51361",
|
||||||
|
"httpApiNotice": true,
|
||||||
|
"welcomeMessage": "1.21.0",
|
||||||
|
"verboseLogging": false,
|
||||||
|
"DANGER_httpHost": null,
|
||||||
|
"DANGER_forceSaveCache": false
|
||||||
|
}
|
||||||
Vendored
+170
File diff suppressed because one or more lines are too long
+14
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"id": "omnisearch",
|
||||||
|
"name": "Omnisearch",
|
||||||
|
"version": "1.29.3",
|
||||||
|
"minAppVersion": "1.7.2",
|
||||||
|
"description": "A search engine that just works.",
|
||||||
|
"author": "Simon Cambier",
|
||||||
|
"authorUrl": "https://scambier.xyz",
|
||||||
|
"fundingUrl": {
|
||||||
|
"Github": "https://github.com/sponsors/scambier",
|
||||||
|
"Ko-fi": "https://ko-fi.com/scambier"
|
||||||
|
},
|
||||||
|
"isDesktopOnly": false
|
||||||
|
}
|
||||||
+150
@@ -0,0 +1,150 @@
|
|||||||
|
.omnisearch-modal {
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result {
|
||||||
|
white-space: normal;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
/* justify-content: space-between; */
|
||||||
|
flex-wrap: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__title-container {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
column-gap: 5px;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__title {
|
||||||
|
white-space: pre-wrap;
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
gap: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__title > span {
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__folder-path {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
gap: 5px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__extension {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__counter {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__body {
|
||||||
|
white-space: normal;
|
||||||
|
font-size: small;
|
||||||
|
word-wrap: normal;
|
||||||
|
|
||||||
|
overflow: hidden;
|
||||||
|
display: -webkit-box;
|
||||||
|
-webkit-line-clamp: 3;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
|
||||||
|
color: var(--text-muted);
|
||||||
|
margin-inline-start: 0.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__embed {
|
||||||
|
margin-left: 1em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__image-container {
|
||||||
|
flex-basis: 20%;
|
||||||
|
text-align: end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-highlight {
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-default-highlight {
|
||||||
|
text-decoration: underline;
|
||||||
|
text-decoration-color: var(--text-highlight-bg);
|
||||||
|
text-decoration-thickness: 3px;
|
||||||
|
text-underline-offset: -1px;
|
||||||
|
text-decoration-skip-ink: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-input-container {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
flex-direction: row;
|
||||||
|
gap: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__icon {
|
||||||
|
display: inline-block;
|
||||||
|
vertical-align: middle;
|
||||||
|
width: 16px;
|
||||||
|
height: 16px;
|
||||||
|
margin-right: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__icon svg {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result__icon--emoji {
|
||||||
|
font-size: 16px;
|
||||||
|
vertical-align: middle;
|
||||||
|
margin-right: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media only screen and (max-width: 600px) {
|
||||||
|
.omnisearch-input-container {
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-input-container__buttons {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
width: 100%;
|
||||||
|
padding: 0 1em 0 1em;
|
||||||
|
gap: 1em;
|
||||||
|
}
|
||||||
|
.omnisearch-input-container__buttons > button {
|
||||||
|
flex-grow: 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media only screen and (min-width: 600px) {
|
||||||
|
.omnisearch-input-container__buttons {
|
||||||
|
margin-inline-end: 1em;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-input-field {
|
||||||
|
position: relative;
|
||||||
|
flex-grow: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result-highlight .cm-content ::selection,
|
||||||
|
.omnisearch-result-highlight .cm-line::selection {
|
||||||
|
background-color: var(
|
||||||
|
--omnisearch-highlight-color,
|
||||||
|
rgba(222, 183, 110, 1)
|
||||||
|
) !important;
|
||||||
|
color: var(--omnisearch-highlight-text-color, black) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.omnisearch-result-highlight .cm-selectionLayer .cm-selectionBackground {
|
||||||
|
background-color: var(
|
||||||
|
--omnisearch-highlight-color,
|
||||||
|
rgba(222, 183, 110, 1)
|
||||||
|
) !important;
|
||||||
|
}
|
||||||
+512
@@ -0,0 +1,512 @@
|
|||||||
|
/*
|
||||||
|
THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
|
||||||
|
if you want to view the source, please visit the github repository of this plugin
|
||||||
|
*/
|
||||||
|
|
||||||
|
var __create = Object.create;
|
||||||
|
var __defProp = Object.defineProperty;
|
||||||
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
||||||
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
||||||
|
var __getProtoOf = Object.getPrototypeOf;
|
||||||
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
||||||
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
||||||
|
var __markAsModule = (target) => __defProp(target, "__esModule", { value: true });
|
||||||
|
var __export = (target, all) => {
|
||||||
|
__markAsModule(target);
|
||||||
|
for (var name in all)
|
||||||
|
__defProp(target, name, { get: all[name], enumerable: true });
|
||||||
|
};
|
||||||
|
var __reExport = (target, module2, desc) => {
|
||||||
|
if (module2 && typeof module2 === "object" || typeof module2 === "function") {
|
||||||
|
for (let key of __getOwnPropNames(module2))
|
||||||
|
if (!__hasOwnProp.call(target, key) && key !== "default")
|
||||||
|
__defProp(target, key, { get: () => module2[key], enumerable: !(desc = __getOwnPropDesc(module2, key)) || desc.enumerable });
|
||||||
|
}
|
||||||
|
return target;
|
||||||
|
};
|
||||||
|
var __toModule = (module2) => {
|
||||||
|
return __reExport(__markAsModule(__defProp(module2 != null ? __create(__getProtoOf(module2)) : {}, "default", module2 && module2.__esModule && "default" in module2 ? { get: () => module2.default, enumerable: true } : { value: module2, enumerable: true })), module2);
|
||||||
|
};
|
||||||
|
var __publicField = (obj, key, value) => {
|
||||||
|
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
// src/main.ts
|
||||||
|
__export(exports, {
|
||||||
|
default: () => SettingsSearch
|
||||||
|
});
|
||||||
|
var import_obsidian = __toModule(require("obsidian"));
|
||||||
|
|
||||||
|
// node_modules/monkey-around/mjs/index.js
|
||||||
|
function around(obj, factories) {
|
||||||
|
const removers = Object.keys(factories).map((key) => around1(obj, key, factories[key]));
|
||||||
|
return removers.length === 1 ? removers[0] : function() {
|
||||||
|
removers.forEach((r) => r());
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function around1(obj, method, createWrapper) {
|
||||||
|
const original = obj[method], hadOwn = obj.hasOwnProperty(method);
|
||||||
|
let current = createWrapper(original);
|
||||||
|
if (original)
|
||||||
|
Object.setPrototypeOf(current, original);
|
||||||
|
Object.setPrototypeOf(wrapper, current);
|
||||||
|
obj[method] = wrapper;
|
||||||
|
return remove;
|
||||||
|
function wrapper(...args) {
|
||||||
|
if (current === original && obj[method] === wrapper)
|
||||||
|
remove();
|
||||||
|
return current.apply(this, args);
|
||||||
|
}
|
||||||
|
function remove() {
|
||||||
|
if (obj[method] === wrapper) {
|
||||||
|
if (hadOwn)
|
||||||
|
obj[method] = original;
|
||||||
|
else
|
||||||
|
delete obj[method];
|
||||||
|
}
|
||||||
|
if (current === original)
|
||||||
|
return;
|
||||||
|
current = original;
|
||||||
|
Object.setPrototypeOf(wrapper, original || Function);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// src/main.ts
|
||||||
|
var SettingsSearch = class extends import_obsidian.Plugin {
|
||||||
|
constructor() {
|
||||||
|
super(...arguments);
|
||||||
|
__publicField(this, "settingsSearchEl", createDiv("settings-search-container vertical-tab-header-group"));
|
||||||
|
__publicField(this, "settingsResultsContainerEl", createDiv("settings-search-results-container vertical-tab-content"));
|
||||||
|
__publicField(this, "settingsNavItemContainer", this.settingsSearchEl.createDiv("vertical-tab-header-group-items").createDiv("vertical-tab-nav-item settings-search-input"));
|
||||||
|
__publicField(this, "settingsResultsEl");
|
||||||
|
__publicField(this, "search");
|
||||||
|
__publicField(this, "locale");
|
||||||
|
__publicField(this, "resources", []);
|
||||||
|
__publicField(this, "results", []);
|
||||||
|
__publicField(this, "loaded", false);
|
||||||
|
__publicField(this, "tabIndex", 0);
|
||||||
|
__publicField(this, "pluginTabIndex", 0);
|
||||||
|
__publicField(this, "seen", []);
|
||||||
|
__publicField(this, "settingCache", new Map());
|
||||||
|
__publicField(this, "searchAppended", false);
|
||||||
|
__publicField(this, "activeIndex", -1);
|
||||||
|
__publicField(this, "activeSetting");
|
||||||
|
__publicField(this, "scope", new import_obsidian.Scope(this.app.scope));
|
||||||
|
__publicField(this, "mobileContainers", []);
|
||||||
|
}
|
||||||
|
async onload() {
|
||||||
|
(window["SettingsSearch"] = {
|
||||||
|
addResources: this.addResources.bind(this),
|
||||||
|
removeResources: this.removeResources.bind(this),
|
||||||
|
removeTabResources: this.removeTabResources.bind(this)
|
||||||
|
}) && this.register(() => delete window["SettingsSearch"]);
|
||||||
|
this.app.workspace.onLayoutReady(async () => {
|
||||||
|
this.settingsResultsContainerEl.createEl("h3", {
|
||||||
|
text: "Settings Search Results"
|
||||||
|
});
|
||||||
|
this.settingsResultsEl = this.settingsResultsContainerEl.createDiv("settings-search-results");
|
||||||
|
this.buildScope();
|
||||||
|
this.buildSearch();
|
||||||
|
this.buildResources();
|
||||||
|
this.buildPluginResources();
|
||||||
|
this.patchSettings();
|
||||||
|
this.loaded = true;
|
||||||
|
this.app.workspace.trigger("settings-search-loaded");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
buildResources() {
|
||||||
|
const tab = this.app.setting.settingTabs[this.tabIndex];
|
||||||
|
if (tab && tab.id !== void 0 && !this.seen.includes(tab.id)) {
|
||||||
|
this.getTabResources(tab);
|
||||||
|
this.tabIndex++;
|
||||||
|
setTimeout(() => this.buildResources());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buildPluginResources() {
|
||||||
|
const tab = this.app.setting.pluginTabs[this.pluginTabIndex];
|
||||||
|
if (tab) {
|
||||||
|
this.getTabResources(tab);
|
||||||
|
this.pluginTabIndex++;
|
||||||
|
setTimeout(() => this.buildPluginResources());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
get manifests() {
|
||||||
|
return Object.values(this.app.plugins.manifests);
|
||||||
|
}
|
||||||
|
addResourceToCache(resource) {
|
||||||
|
if (!resource || !resource.text || !resource.name || !resource.tab) {
|
||||||
|
return new Error("A valid resource must be provided.");
|
||||||
|
}
|
||||||
|
let name;
|
||||||
|
if (resource.external) {
|
||||||
|
name = createFragment((el) => {
|
||||||
|
(0, import_obsidian.setIcon)(el.createSpan({
|
||||||
|
attr: {
|
||||||
|
"aria-label": "This setting was added by another plugin."
|
||||||
|
}
|
||||||
|
}), "info");
|
||||||
|
el.createSpan({ text: resource.text });
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
name = resource.text;
|
||||||
|
}
|
||||||
|
const setting = new import_obsidian.Setting(createDiv()).setName(name).setDesc(createFragment((e) => e.createDiv().innerHTML = resource.desc ?? ""));
|
||||||
|
if (resource.external) {
|
||||||
|
setting.settingEl.addClass("set-externally");
|
||||||
|
}
|
||||||
|
if (resource.tab == "community-plugins") {
|
||||||
|
let plugin = this.manifests.find((p) => p.name == resource.text);
|
||||||
|
if (plugin && this.app.plugins.getPlugin(plugin.id)?._loaded && this.app.setting.pluginTabs.find((t) => t.id == plugin.id)) {
|
||||||
|
setting.addExtraButton((b) => {
|
||||||
|
b.setTooltip(`Open ${resource.text} Settings`).onClick(() => {
|
||||||
|
this.app.setting.openTabById(plugin.id);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (resource.tab == "plugins") {
|
||||||
|
const plugins = Object.values(this.app.internalPlugins.plugins);
|
||||||
|
const plugin = plugins.find((p) => p._loaded && p.instance.name == resource.text);
|
||||||
|
if (plugin && this.app.setting.pluginTabs.find((t) => t.id == plugin.instance.id)) {
|
||||||
|
setting.addExtraButton((b) => {
|
||||||
|
b.setTooltip(`Open ${resource.text} Settings`).onClick(() => {
|
||||||
|
this.app.setting.openTabById(plugin.instance.id);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setting.addExtraButton((b) => {
|
||||||
|
b.setIcon("forward-arrow").onClick(() => {
|
||||||
|
this.showResult(resource);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
this.settingCache.set(resource, setting);
|
||||||
|
}
|
||||||
|
getResourceFromCache(resource) {
|
||||||
|
if (!this.settingCache.has(resource)) {
|
||||||
|
this.addResourceToCache(resource);
|
||||||
|
}
|
||||||
|
return this.settingCache.get(resource);
|
||||||
|
}
|
||||||
|
removeResourcesFromCache(resources) {
|
||||||
|
for (const resource of resources) {
|
||||||
|
this.settingCache.delete(resource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
addResources(...resources) {
|
||||||
|
for (const resource of resources) {
|
||||||
|
resource.external = true;
|
||||||
|
if (this.resources.find((k) => this.equivalent(resource, k)))
|
||||||
|
continue;
|
||||||
|
this.resources.push(resource);
|
||||||
|
this.addResourceToCache(resource);
|
||||||
|
}
|
||||||
|
return () => this.removeResources(...resources);
|
||||||
|
}
|
||||||
|
equivalent(resource1, resource2) {
|
||||||
|
return resource1.name == resource2.name && resource1.tab == resource2.tab && resource1.text == resource2.text && resource1.desc == resource2.desc && resource1.external == resource2.external;
|
||||||
|
}
|
||||||
|
removeResources(...resources) {
|
||||||
|
const removing = [];
|
||||||
|
const keys = [...this.settingCache.keys()];
|
||||||
|
for (const resource of resources) {
|
||||||
|
if (!resource || !resource.text || !resource.name || !resource.tab) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
resource.external = true;
|
||||||
|
this.resources = this.resources.filter((r) => !this.equivalent(resource, r));
|
||||||
|
removing.push(...keys.filter((k) => k == resource || this.equivalent(resource, k)));
|
||||||
|
}
|
||||||
|
this.removeResourcesFromCache(removing);
|
||||||
|
}
|
||||||
|
removeTabResources(tab) {
|
||||||
|
const removing = this.resources.filter((t) => t.tab == tab);
|
||||||
|
this.resources = this.resources.filter((t) => t.tab != tab);
|
||||||
|
this.removeResourcesFromCache(removing);
|
||||||
|
}
|
||||||
|
async getTabResources(tab) {
|
||||||
|
await tab.display();
|
||||||
|
const settings = tab.containerEl.querySelectorAll(".setting-item:not(.setting-item-header)");
|
||||||
|
for (const el of Array.from(settings)) {
|
||||||
|
const text = el.querySelector(".setting-item-name")?.textContent;
|
||||||
|
if (!text)
|
||||||
|
continue;
|
||||||
|
const desc = el.querySelector(".setting-item-description")?.innerHTML ?? "";
|
||||||
|
const resource = {
|
||||||
|
tab: tab.id,
|
||||||
|
name: tab.name,
|
||||||
|
text,
|
||||||
|
desc
|
||||||
|
};
|
||||||
|
this.resources.push(resource);
|
||||||
|
this.addResourceToCache(resource);
|
||||||
|
}
|
||||||
|
if (this.app.setting.activeTab?.id == tab.id)
|
||||||
|
return;
|
||||||
|
this.seen.push(tab.id);
|
||||||
|
tab.containerEl.detach();
|
||||||
|
tab.hide();
|
||||||
|
}
|
||||||
|
patchSettings() {
|
||||||
|
const self = this;
|
||||||
|
this.register(around(this.app.setting, {
|
||||||
|
onOpen: function(next) {
|
||||||
|
return function() {
|
||||||
|
next.apply(this);
|
||||||
|
if (!import_obsidian.Platform.isMobile)
|
||||||
|
self.search.inputEl.focus();
|
||||||
|
return next;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
this.register(around(this.app.setting, {
|
||||||
|
addSettingTab: function(next) {
|
||||||
|
return function(tab) {
|
||||||
|
if (tab && tab.id !== void 0 && !self.seen.includes(tab.id)) {
|
||||||
|
self.getTabResources(tab);
|
||||||
|
}
|
||||||
|
return next.call(this, tab);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
this.register(around(this.app.setting, {
|
||||||
|
removeSettingTab: function(next) {
|
||||||
|
return function(tab) {
|
||||||
|
if (this.isPluginSettingTab(tab)) {
|
||||||
|
self.removeTabResources(tab.id);
|
||||||
|
}
|
||||||
|
return next.call(this, tab);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
this.register(around(this.app.setting, {
|
||||||
|
openTab: function(next) {
|
||||||
|
return function(tab) {
|
||||||
|
self.searchAppended = false;
|
||||||
|
self.app.keymap.popScope(self.scope);
|
||||||
|
return next.call(this, tab);
|
||||||
|
};
|
||||||
|
},
|
||||||
|
openTabById: function(next) {
|
||||||
|
return function(tab) {
|
||||||
|
self.searchAppended = false;
|
||||||
|
self.app.keymap.popScope(self.scope);
|
||||||
|
return next.call(this, tab);
|
||||||
|
};
|
||||||
|
},
|
||||||
|
onClose: function(next) {
|
||||||
|
return function() {
|
||||||
|
if (import_obsidian.Platform.isMobile) {
|
||||||
|
self.detach();
|
||||||
|
}
|
||||||
|
return next.call(this);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
buildSearch() {
|
||||||
|
const tempSetting = new import_obsidian.Setting(createDiv()).addSearch((s) => {
|
||||||
|
this.search = s;
|
||||||
|
});
|
||||||
|
this.settingsNavItemContainer.append(tempSetting.controlEl);
|
||||||
|
tempSetting.settingEl.detach();
|
||||||
|
this.search.onChange((v) => {
|
||||||
|
this.onChange(v);
|
||||||
|
});
|
||||||
|
this.search.setPlaceholder("Search settings...");
|
||||||
|
this.app.setting.tabHeadersEl.prepend(this.settingsSearchEl);
|
||||||
|
}
|
||||||
|
buildScope() {
|
||||||
|
this.scope.register(["Ctrl"], "N", () => {
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.activeSetting.settingEl.removeClass("active");
|
||||||
|
}
|
||||||
|
this.activeIndex = ((this.activeIndex + 1) % this.results.length + this.results.length) % this.results.length;
|
||||||
|
this.centerActiveSetting();
|
||||||
|
});
|
||||||
|
this.scope.register([], "ArrowDown", () => {
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.activeSetting.settingEl.removeClass("active");
|
||||||
|
}
|
||||||
|
this.activeIndex = ((this.activeIndex + 1) % this.results.length + this.results.length) % this.results.length;
|
||||||
|
this.centerActiveSetting();
|
||||||
|
});
|
||||||
|
this.scope.register(["Ctrl"], "P", () => {
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.activeSetting.settingEl.removeClass("active");
|
||||||
|
}
|
||||||
|
this.activeIndex = ((this.activeIndex - 1) % this.results.length + this.results.length) % this.results.length;
|
||||||
|
this.centerActiveSetting();
|
||||||
|
});
|
||||||
|
this.scope.register([], "ArrowUp", () => {
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.activeSetting.settingEl.removeClass("active");
|
||||||
|
}
|
||||||
|
this.activeIndex = ((this.activeIndex - 1) % this.results.length + this.results.length) % this.results.length;
|
||||||
|
this.centerActiveSetting();
|
||||||
|
});
|
||||||
|
this.scope.register([], "Enter", () => {
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.showResult(this.results[this.activeIndex]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
centerActiveSetting() {
|
||||||
|
const result = this.results[this.activeIndex];
|
||||||
|
this.activeSetting = this.getResourceFromCache(result);
|
||||||
|
this.activeSetting.settingEl.addClass("active");
|
||||||
|
this.activeSetting.settingEl.scrollIntoView({
|
||||||
|
behavior: "auto",
|
||||||
|
block: "nearest"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
detachFromMobile() {
|
||||||
|
if (import_obsidian.Platform.isMobile) {
|
||||||
|
this.settingsResultsContainerEl.detach();
|
||||||
|
for (const header of this.mobileContainers) {
|
||||||
|
this.app.setting.tabHeadersEl.append(header);
|
||||||
|
}
|
||||||
|
this.search.setValue("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
detachFromDesktop() {
|
||||||
|
if (import_obsidian.Platform.isDesktop) {
|
||||||
|
this.app.setting.openTabById(this.app.setting.lastTabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
detach() {
|
||||||
|
this.detachFromDesktop();
|
||||||
|
this.detachFromMobile();
|
||||||
|
this.searchAppended = false;
|
||||||
|
}
|
||||||
|
onChange(v) {
|
||||||
|
if (!v) {
|
||||||
|
this.detach();
|
||||||
|
this.app.keymap.popScope(this.scope);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!this.searchAppended) {
|
||||||
|
this.activeIndex = -1;
|
||||||
|
this.app.keymap.popScope(this.scope);
|
||||||
|
this.app.keymap.pushScope(this.scope);
|
||||||
|
if (this.activeSetting) {
|
||||||
|
this.activeSetting.settingEl.removeClass("active");
|
||||||
|
this.activeSetting = null;
|
||||||
|
}
|
||||||
|
if (!import_obsidian.Platform.isMobile) {
|
||||||
|
this.app.setting.activeTab.navEl.removeClass("is-active");
|
||||||
|
this.app.setting.tabContentContainer.empty();
|
||||||
|
this.app.setting.tabContentContainer.append(this.settingsResultsContainerEl);
|
||||||
|
} else {
|
||||||
|
const headers = this.app.setting.tabHeadersEl.querySelectorAll(".vertical-tab-header-group:not(.settings-search-container)");
|
||||||
|
for (const header of Array.from(headers)) {
|
||||||
|
this.mobileContainers.push(header);
|
||||||
|
header.detach();
|
||||||
|
}
|
||||||
|
this.app.setting.tabHeadersEl.append(this.settingsResultsContainerEl);
|
||||||
|
}
|
||||||
|
this.searchAppended = true;
|
||||||
|
}
|
||||||
|
this.appendResults(this.performFuzzySearch(v));
|
||||||
|
}
|
||||||
|
getMatchText(text, result) {
|
||||||
|
const matchElements = {};
|
||||||
|
return createFragment((content) => {
|
||||||
|
for (let i = 0; i < text.length; i++) {
|
||||||
|
let match = result.matches.find((m) => m[0] === i);
|
||||||
|
if (match) {
|
||||||
|
const index = result.matches.indexOf(match);
|
||||||
|
if (!matchElements[index]) {
|
||||||
|
matchElements[index] = createSpan("suggestion-highlight");
|
||||||
|
}
|
||||||
|
let element = matchElements[index];
|
||||||
|
content.appendChild(element);
|
||||||
|
element.appendText(text.substring(match[0], match[1]));
|
||||||
|
i += match[1] - match[0] - 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
content.appendText(text[i]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
appendResults(results) {
|
||||||
|
this.settingsResultsEl.empty();
|
||||||
|
if (results.length) {
|
||||||
|
const headers = {};
|
||||||
|
for (const resource of results) {
|
||||||
|
if (!(resource.tab in headers)) {
|
||||||
|
headers[resource.tab] = this.settingsResultsEl.createDiv();
|
||||||
|
new import_obsidian.Setting(headers[resource.tab]).setHeading().setName(resource.name);
|
||||||
|
}
|
||||||
|
const setting = this.getResourceFromCache(resource);
|
||||||
|
headers[resource.tab].append(setting.settingEl);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.settingsResultsEl.setText("No results found :(");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
showResult(result) {
|
||||||
|
this.search.setValue("");
|
||||||
|
const tab = this.app.setting.settingTabs.find((t) => t.id == result.tab) ?? this.app.setting.pluginTabs.find((t) => t.id == result.tab);
|
||||||
|
if (!tab) {
|
||||||
|
new import_obsidian.Notice("There was an issue opening the setting tab.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.app.setting.openTabById(tab.id);
|
||||||
|
this.app.keymap.popScope(this.scope);
|
||||||
|
this.detach();
|
||||||
|
try {
|
||||||
|
const names = tab.containerEl.querySelectorAll(".setting-item-name");
|
||||||
|
const el = Array.from(names).find((n) => n.textContent == result.text);
|
||||||
|
if (!el)
|
||||||
|
return;
|
||||||
|
const setting = el.closest(".setting-item");
|
||||||
|
if (!setting)
|
||||||
|
return;
|
||||||
|
if (tab.id == "obsidian-style-settings") {
|
||||||
|
let collapsed = setting.closest(".style-settings-container");
|
||||||
|
let previous = collapsed?.previousElementSibling;
|
||||||
|
while (previous != null && previous.hasClass("is-collapsed") && previous.hasClass("style-settings-heading")) {
|
||||||
|
previous.removeClass("is-collapsed");
|
||||||
|
collapsed = collapsed.parentElement?.closest(".style-settings-container");
|
||||||
|
previous = collapsed?.previousElementSibling;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let details = setting.closest("details");
|
||||||
|
while (details) {
|
||||||
|
details.setAttr("open", "open");
|
||||||
|
details = details.parentElement?.closest("details");
|
||||||
|
}
|
||||||
|
setting.scrollIntoView(true);
|
||||||
|
setting.addClass("is-flashing");
|
||||||
|
window.setTimeout(() => setting.removeClass("is-flashing"), 3e3);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
performFuzzySearch(input) {
|
||||||
|
const results = [], hotkeys = [];
|
||||||
|
for (const resource of this.resources) {
|
||||||
|
let result = (0, import_obsidian.prepareSimpleSearch)(input)(resource.text) ?? (0, import_obsidian.prepareSimpleSearch)(input)(resource.desc);
|
||||||
|
if (result) {
|
||||||
|
if (resource.tab == "hotkeys") {
|
||||||
|
hotkeys.push(resource);
|
||||||
|
} else {
|
||||||
|
results.push(resource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.results = [...results, ...hotkeys];
|
||||||
|
return this.results;
|
||||||
|
}
|
||||||
|
onunload() {
|
||||||
|
this.settingsSearchEl.detach();
|
||||||
|
this.settingsResultsEl.detach();
|
||||||
|
this.detach();
|
||||||
|
if (this.searchAppended && import_obsidian.Platform.isDesktop)
|
||||||
|
this.app.setting.openTabById(this.app.setting.lastTabId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/* nosourcemap */
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"id": "settings-search",
|
||||||
|
"name": "Settings Search",
|
||||||
|
"version": "1.3.10",
|
||||||
|
"minAppVersion": "0.12.17",
|
||||||
|
"author": "Jeremy Valentine",
|
||||||
|
"description": "Globally search settings in Obsidian.md",
|
||||||
|
"authorUrl": "https://github.com/valentine195",
|
||||||
|
"isDesktopOnly": false
|
||||||
|
}
|
||||||
+25
@@ -0,0 +1,25 @@
|
|||||||
|
/* src/assets/main.css */
|
||||||
|
.settings-search-container.vertical-tab-header-group {
|
||||||
|
padding-bottom: 0;
|
||||||
|
}
|
||||||
|
.settings-search-input {
|
||||||
|
padding-left: 6px;
|
||||||
|
}
|
||||||
|
.vertical-tab-nav-item.settings-search-input {
|
||||||
|
background-color: inherit !important;
|
||||||
|
}
|
||||||
|
.settings-search-input .setting-item-control {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
.settings-search-input .search-input-container {
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
.settings-search-results .setting-item.active {
|
||||||
|
background-color: var(--background-secondary);
|
||||||
|
}
|
||||||
|
.settings-search-results .set-externally .setting-item-name {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* src/styles.css */
|
||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"folder": "10 Knowledge/Templates"
|
||||||
|
}
|
||||||
Vendored
+24
-11
@@ -4,21 +4,21 @@
|
|||||||
"type": "split",
|
"type": "split",
|
||||||
"children": [
|
"children": [
|
||||||
{
|
{
|
||||||
"id": "cc39b090a168e0a9",
|
"id": "599535c394bd0ccc",
|
||||||
"type": "tabs",
|
"type": "tabs",
|
||||||
"children": [
|
"children": [
|
||||||
{
|
{
|
||||||
"id": "99ffb205f7f48424",
|
"id": "84655ec8deead6f8",
|
||||||
"type": "leaf",
|
"type": "leaf",
|
||||||
"state": {
|
"state": {
|
||||||
"type": "markdown",
|
"type": "markdown",
|
||||||
"state": {
|
"state": {
|
||||||
"file": "Welcome.md",
|
"file": "20 Work/Ideas/Giving everyone a chance to learn management.md",
|
||||||
"mode": "source",
|
"mode": "source",
|
||||||
"source": false
|
"source": false
|
||||||
},
|
},
|
||||||
"icon": "lucide-file",
|
"icon": "lucide-file",
|
||||||
"title": "Welcome"
|
"title": "Giving everyone a chance to learn management"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
@@ -78,7 +78,7 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"direction": "horizontal",
|
"direction": "horizontal",
|
||||||
"width": 300
|
"width": 402.5
|
||||||
},
|
},
|
||||||
"right": {
|
"right": {
|
||||||
"id": "d9252c3a3a14343a",
|
"id": "d9252c3a3a14343a",
|
||||||
@@ -192,13 +192,28 @@
|
|||||||
"command-palette:Open command palette": false,
|
"command-palette:Open command palette": false,
|
||||||
"bases:Create new base": false,
|
"bases:Create new base": false,
|
||||||
"obsidian-git:Open Git source control": false,
|
"obsidian-git:Open Git source control": false,
|
||||||
"iconic:Open rulebook": false
|
"iconic:Open rulebook": false,
|
||||||
|
"omnisearch:Omnisearch": false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"active": "99ffb205f7f48424",
|
"active": "84655ec8deead6f8",
|
||||||
"lastOpenFiles": [
|
"lastOpenFiles": [
|
||||||
"10 Knowledge/CTO Academy.md",
|
"20 Work/Ideas/Coaching on management.md",
|
||||||
|
"20 Work/Ideas/Giving everyone a chance to learn management.md",
|
||||||
|
"10 Knowledge/Templates/Journal Template.md",
|
||||||
|
"10 Knowledge/Templates/Academy Template.md",
|
||||||
|
"10 Knowledge/Templates/Employe - Team Template.md",
|
||||||
|
"10 Knowledge/Templates/Project Template.md",
|
||||||
|
"10 Knowledge/Templates/Idea Template.md",
|
||||||
|
"10 Knowledge/Templates/Meeting Template.md",
|
||||||
|
"10 Knowledge/Templates/Decision Template.md",
|
||||||
|
"10 Knowledge/CTO Academy/Data/ETL vs ELT.md",
|
||||||
|
"10 Knowledge/Templates",
|
||||||
|
"10 Knowledge/CTO Academy/Data/Data Lake vs Data Warehouse vs Lakehouse.md",
|
||||||
|
"10 Knowledge/CTO Academy/Data/Medallion Architecture.md",
|
||||||
"Welcome.md",
|
"Welcome.md",
|
||||||
|
"10 Knowledge/CTO Academy/Data",
|
||||||
|
"10 Knowledge/CTO Academy.md",
|
||||||
"CTO Academy",
|
"CTO Academy",
|
||||||
"Executive Playbook.md",
|
"Executive Playbook.md",
|
||||||
"CTO Academy.md",
|
"CTO Academy.md",
|
||||||
@@ -209,8 +224,6 @@
|
|||||||
"40 Journal",
|
"40 Journal",
|
||||||
"30 Personal Projects/Modji",
|
"30 Personal Projects/Modji",
|
||||||
"30 Personal Projects",
|
"30 Personal Projects",
|
||||||
"20 Work/Ideas",
|
"20 Work/Ideas"
|
||||||
"20 Work/Vendors",
|
|
||||||
"20 Work/KPIs"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,499 @@
|
|||||||
|
|
||||||
|
**Time: ~10–15 minutes**
|
||||||
|
|
||||||
|
Today's goal is that if someone says:
|
||||||
|
|
||||||
|
> "The data is in the lake."
|
||||||
|
|
||||||
|
or:
|
||||||
|
|
||||||
|
> "We're moving to a lakehouse."
|
||||||
|
|
||||||
|
or:
|
||||||
|
|
||||||
|
> "That's handled by Synapse."
|
||||||
|
|
||||||
|
you immediately understand what category of thing they're talking about.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1. Start with the simplest mental model
|
||||||
|
|
||||||
|
Think of these as answering different questions:
|
||||||
|
|
||||||
|
|Concept|Think of it as|
|
||||||
|
|---|---|
|
||||||
|
|**Data Lake**|A place to store huge amounts of data|
|
||||||
|
|**Data Warehouse**|A structured system optimized for analytics|
|
||||||
|
|**Lakehouse**|An architecture combining characteristics of both|
|
||||||
|
|**Azure Synapse**|A Microsoft analytics platform providing tools to work with data|
|
||||||
|
|
||||||
|
The first three describe **data architectures/storage approaches**.
|
||||||
|
|
||||||
|
Synapse is a **product/platform**.
|
||||||
|
|
||||||
|
That's the first distinction to remember.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2. Data Lake 🌊
|
||||||
|
|
||||||
|
A Data Lake is designed to store **large quantities of data in many forms**.
|
||||||
|
|
||||||
|
You might put:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GA4 event exports
|
||||||
|
JW Player logs
|
||||||
|
JSON files
|
||||||
|
CSV files
|
||||||
|
Parquet files
|
||||||
|
Application logs
|
||||||
|
Images
|
||||||
|
Video metadata
|
||||||
|
IoT data
|
||||||
|
Database exports
|
||||||
|
```
|
||||||
|
|
||||||
|
into a Data Lake.
|
||||||
|
|
||||||
|
The important thing is that the data doesn't necessarily need to be modeled for business reporting before you store it.
|
||||||
|
|
||||||
|
Think:
|
||||||
|
|
||||||
|
> **Store first. Figure out how to use it later.**
|
||||||
|
|
||||||
|
For your company, something like Azure Data Lake Storage could contain enormous quantities of raw and processed data.
|
||||||
|
|
||||||
|
You could potentially organize it:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Data Lake
|
||||||
|
|
||||||
|
├── bronze/
|
||||||
|
│ ├── ga4/
|
||||||
|
│ ├── jwplayer/
|
||||||
|
│ └── ott/
|
||||||
|
│
|
||||||
|
├── silver/
|
||||||
|
│ ├── audience/
|
||||||
|
│ └── content/
|
||||||
|
│
|
||||||
|
└── gold/
|
||||||
|
├── audience_metrics/
|
||||||
|
└── content_performance/
|
||||||
|
```
|
||||||
|
|
||||||
|
So remember our previous lesson:
|
||||||
|
|
||||||
|
**A Data Lake can contain Bronze, Silver AND Gold.**
|
||||||
|
|
||||||
|
The Lake is **where/how data is stored**.
|
||||||
|
|
||||||
|
The Medallion layers describe **the maturity of the data**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3. Data Warehouse 🏢
|
||||||
|
|
||||||
|
A Data Warehouse is much more structured.
|
||||||
|
|
||||||
|
It's designed primarily for:
|
||||||
|
|
||||||
|
> **Analytics and reporting.**
|
||||||
|
|
||||||
|
Instead of throwing everything into it, you intentionally organize data for analysis.
|
||||||
|
|
||||||
|
Imagine:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DIM_DATE
|
||||||
|
│
|
||||||
|
│
|
||||||
|
DIM_CONTENT ─── FACT_VIEWING ─── DIM_USER
|
||||||
|
│
|
||||||
|
│
|
||||||
|
DIM_PLATFORM
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you can easily ask:
|
||||||
|
|
||||||
|
> How many hours were watched...
|
||||||
|
|
||||||
|
> by content...
|
||||||
|
|
||||||
|
> by platform...
|
||||||
|
|
||||||
|
> by month...
|
||||||
|
|
||||||
|
> by audience?
|
||||||
|
|
||||||
|
That's what warehouses are excellent at.
|
||||||
|
|
||||||
|
You'll eventually hear terms such as:
|
||||||
|
|
||||||
|
- Fact tables
|
||||||
|
|
||||||
|
- Dimension tables
|
||||||
|
|
||||||
|
- Star schemas
|
||||||
|
|
||||||
|
- Snowflake schemas
|
||||||
|
|
||||||
|
|
||||||
|
Those are closely associated with analytical data modeling and warehouses.
|
||||||
|
|
||||||
|
We'll cover those soon.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4. Data Lake vs Data Warehouse
|
||||||
|
|
||||||
|
The traditional distinction looks roughly like this:
|
||||||
|
|
||||||
|
||Data Lake|Data Warehouse|
|
||||||
|
|---|---|---|
|
||||||
|
|Data|Raw + processed|Primarily structured|
|
||||||
|
|Structure|Flexible|Highly organized|
|
||||||
|
|Cost|Generally cheaper storage|Generally more expensive compute/storage|
|
||||||
|
|Purpose|Many possible uses|Analytics/reporting|
|
||||||
|
|Users|Data engineers/scientists|Analysts/BI|
|
||||||
|
|Schema|Often applied later|Typically defined deliberately|
|
||||||
|
|
||||||
|
A simplified architecture could therefore be:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SOURCE SYSTEMS
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
DATA LAKE
|
||||||
|
│
|
||||||
|
Bronze
|
||||||
|
│
|
||||||
|
Silver
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
DATA WAREHOUSE
|
||||||
|
│
|
||||||
|
Gold
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
Again, that's **one possible architecture**, not a universal rule.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5. The problem
|
||||||
|
|
||||||
|
Companies started asking:
|
||||||
|
|
||||||
|
> Why are we maintaining two completely separate worlds?
|
||||||
|
|
||||||
|
They had:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Data Lake
|
||||||
|
|
||||||
|
AND
|
||||||
|
|
||||||
|
Data Warehouse
|
||||||
|
```
|
||||||
|
|
||||||
|
The lake was flexible and inexpensive.
|
||||||
|
|
||||||
|
The warehouse was structured and excellent for analytics.
|
||||||
|
|
||||||
|
But moving data between them added:
|
||||||
|
|
||||||
|
- complexity
|
||||||
|
|
||||||
|
- duplication
|
||||||
|
|
||||||
|
- pipelines
|
||||||
|
|
||||||
|
- governance challenges
|
||||||
|
|
||||||
|
- additional cost
|
||||||
|
|
||||||
|
|
||||||
|
This contributed to the rise of the:
|
||||||
|
|
||||||
|
# Lakehouse 🏠🌊
|
||||||
|
|
||||||
|
The idea is:
|
||||||
|
|
||||||
|
> **Can we get warehouse-like capabilities directly on top of lake-style storage?**
|
||||||
|
|
||||||
|
Conceptually:
|
||||||
|
|
||||||
|
```text
|
||||||
|
LAKEHOUSE
|
||||||
|
|
||||||
|
┌────────────────────────────┐
|
||||||
|
│ │
|
||||||
|
│ Bronze │
|
||||||
|
│ ↓ │
|
||||||
|
│ Silver │
|
||||||
|
│ ↓ │
|
||||||
|
│ Gold │
|
||||||
|
│ │
|
||||||
|
│ Structured + Unstructured │
|
||||||
|
│ │
|
||||||
|
│ BI + ML + Data Science │
|
||||||
|
│ │
|
||||||
|
└────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
Technologies such as Delta Lake helped make this architecture practical by adding database-like capabilities to data stored in a lake.
|
||||||
|
|
||||||
|
The big idea:
|
||||||
|
|
||||||
|
> **Data Lake flexibility + Data Warehouse capabilities = Lakehouse**
|
||||||
|
|
||||||
|
That's simplified, but it's the mental model I want you to remember.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6. Where does Azure Synapse fit?
|
||||||
|
|
||||||
|
This is where people get confused.
|
||||||
|
|
||||||
|
Azure Synapse Analytics is not simply a Data Lake.
|
||||||
|
|
||||||
|
It's not simply a Data Warehouse either.
|
||||||
|
|
||||||
|
It's a Microsoft **analytics platform**.
|
||||||
|
|
||||||
|
It provides capabilities around things such as:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Data
|
||||||
|
↓
|
||||||
|
Pipelines
|
||||||
|
↓
|
||||||
|
Transformations
|
||||||
|
↓
|
||||||
|
SQL
|
||||||
|
↓
|
||||||
|
Spark
|
||||||
|
↓
|
||||||
|
Analytics
|
||||||
|
```
|
||||||
|
|
||||||
|
It can interact with services such as Azure Data Lake Storage and provide SQL and Spark-based analytics capabilities.
|
||||||
|
|
||||||
|
So someone saying:
|
||||||
|
|
||||||
|
> "Our data is in Synapse."
|
||||||
|
|
||||||
|
is a little like saying:
|
||||||
|
|
||||||
|
> "Our website is in AWS."
|
||||||
|
|
||||||
|
Okay...
|
||||||
|
|
||||||
|
But **where and how?**
|
||||||
|
|
||||||
|
The CTO question becomes:
|
||||||
|
|
||||||
|
> "Can you show me how we're using Synapse in our architecture?"
|
||||||
|
|
||||||
|
Maybe they're using:
|
||||||
|
|
||||||
|
- Synapse pipelines for data movement.
|
||||||
|
|
||||||
|
- Spark for transformations.
|
||||||
|
|
||||||
|
- Serverless SQL to query files in the Data Lake.
|
||||||
|
|
||||||
|
- Dedicated SQL pools as a warehouse.
|
||||||
|
|
||||||
|
|
||||||
|
Those are very different things.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 7. Connecting Lessons 1 and 2
|
||||||
|
|
||||||
|
Now we have two separate dimensions.
|
||||||
|
|
||||||
|
### Data maturity
|
||||||
|
|
||||||
|
```text
|
||||||
|
Bronze
|
||||||
|
↓
|
||||||
|
Silver
|
||||||
|
↓
|
||||||
|
Gold
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data architecture
|
||||||
|
|
||||||
|
```text
|
||||||
|
Data Lake
|
||||||
|
|
||||||
|
Data Warehouse
|
||||||
|
|
||||||
|
Lakehouse
|
||||||
|
```
|
||||||
|
|
||||||
|
And then we have a product:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Azure Synapse
|
||||||
|
```
|
||||||
|
|
||||||
|
These concepts can overlap.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Azure Platform
|
||||||
|
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
|
||||||
|
Azure Data Lake Storage
|
||||||
|
|
||||||
|
Bronze → Silver → Gold
|
||||||
|
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
|
||||||
|
Synapse SQL
|
||||||
|
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
That's a perfectly plausible architecture.
|
||||||
|
|
||||||
|
But your actual company architecture could be different.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 8. CTO perspective
|
||||||
|
|
||||||
|
Imagine you're in a meeting and someone says:
|
||||||
|
|
||||||
|
> "We need a Lakehouse."
|
||||||
|
|
||||||
|
Don't immediately say yes.
|
||||||
|
|
||||||
|
Ask:
|
||||||
|
|
||||||
|
> **"What problem are we solving that our current Data Lake and Warehouse architecture can't solve?"**
|
||||||
|
|
||||||
|
That's the executive question.
|
||||||
|
|
||||||
|
Because "Lakehouse" sounds modern.
|
||||||
|
|
||||||
|
That doesn't automatically make it necessary.
|
||||||
|
|
||||||
|
Then ask:
|
||||||
|
|
||||||
|
> "Where does our Gold layer currently live?"
|
||||||
|
|
||||||
|
> "Are we duplicating data between our Lake and Warehouse?"
|
||||||
|
|
||||||
|
> "What's driving our current cost?"
|
||||||
|
|
||||||
|
> "What workloads are we supporting: BI, data science, ML, or all three?"
|
||||||
|
|
||||||
|
> "How is data governed across these environments?"
|
||||||
|
|
||||||
|
Now you're participating in an architecture discussion without needing to write a single Spark job.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 9. Meeting scenario
|
||||||
|
|
||||||
|
A data architect tells you:
|
||||||
|
|
||||||
|
> "We should migrate our architecture to a Lakehouse."
|
||||||
|
|
||||||
|
A weaker response would be:
|
||||||
|
|
||||||
|
> "Okay, what's involved?"
|
||||||
|
|
||||||
|
A stronger CTO response:
|
||||||
|
|
||||||
|
> **"Before we discuss the technology, I'd like to understand the problem we're solving. Are we trying to reduce duplication between the Lake and Warehouse, improve performance, lower cost, or support new workloads? Once we agree on the problem, we can determine whether a Lakehouse is actually the right architecture."**
|
||||||
|
|
||||||
|
Notice our executive communication training appearing again?
|
||||||
|
|
||||||
|
**Control the structure.**
|
||||||
|
|
||||||
|
You just changed:
|
||||||
|
|
||||||
|
> Should we implement a Lakehouse?
|
||||||
|
|
||||||
|
into:
|
||||||
|
|
||||||
|
> What problem are we trying to solve?
|
||||||
|
|
||||||
|
That's exactly the skill you told me you want to develop.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 10. Your cheat sheet
|
||||||
|
|
||||||
|
Remember these four sentences:
|
||||||
|
|
||||||
|
> 🌊 **Data Lake:** Store large amounts of diverse data flexibly.
|
||||||
|
|
||||||
|
> 🏢 **Data Warehouse:** Organize structured data for analytics and reporting.
|
||||||
|
|
||||||
|
> 🏠 **Lakehouse:** Bring warehouse-like capabilities to lake-style storage.
|
||||||
|
|
||||||
|
> 🧰 **Azure Synapse:** Microsoft's analytics platform that provides multiple tools for working with and analyzing data.
|
||||||
|
|
||||||
|
And remember:
|
||||||
|
|
||||||
|
> **Bronze/Silver/Gold tells me how refined the data is.**
|
||||||
|
|
||||||
|
> **Lake/Warehouse/Lakehouse tells me about the architecture.**
|
||||||
|
|
||||||
|
> **Synapse tells me about the technology/platform being used.**
|
||||||
|
|
||||||
|
That's the distinction I want you to walk away with tonight.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Obsidian
|
||||||
|
|
||||||
|
Create:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Knowledge
|
||||||
|
└── CTO Academy
|
||||||
|
└── Data
|
||||||
|
└── Data Lake vs Data Warehouse vs Lakehouse.md
|
||||||
|
```
|
||||||
|
|
||||||
|
At the bottom:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
## Related Concepts
|
||||||
|
|
||||||
|
- [[Medallion Architecture]]
|
||||||
|
- [[Azure Synapse]]
|
||||||
|
- [[ETL vs ELT]]
|
||||||
|
- [[Star Schema]]
|
||||||
|
- [[Fact Table]]
|
||||||
|
- [[Dimension Table]]
|
||||||
|
- [[Data Lake]]
|
||||||
|
- [[Data Warehouse]]
|
||||||
|
- [[Lakehouse]]
|
||||||
|
```
|
||||||
|
|
||||||
|
Don't create individual pages for all of them yet.
|
||||||
|
|
||||||
|
**Lesson 3 should be ETL vs ELT.** That's the missing bridge between today's lesson and the first lesson: _how does the data actually travel from the source through Bronze, Silver and Gold?_
|
||||||
@@ -0,0 +1,757 @@
|
|||||||
|
## How Data Moves Through the Architecture
|
||||||
|
|
||||||
|
**Time: ~10–15 minutes**
|
||||||
|
|
||||||
|
Today we're connecting [[Medallion Architecture]] and [[Data Lake vs Data Warehouse vs Lakehouse]]
|
||||||
|
.
|
||||||
|
|
||||||
|
You now understand:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Lesson 1
|
||||||
|
Bronze → Silver → Gold
|
||||||
|
Data maturity
|
||||||
|
|
||||||
|
Lesson 2
|
||||||
|
Data Lake → Data Warehouse → Lakehouse
|
||||||
|
Data architecture
|
||||||
|
|
||||||
|
Lesson 3
|
||||||
|
ETL / ELT
|
||||||
|
How data gets there and gets transformed
|
||||||
|
```
|
||||||
|
|
||||||
|
By the end of this lesson, when someone says:
|
||||||
|
|
||||||
|
> "We're using an ELT pipeline."
|
||||||
|
|
||||||
|
you should understand exactly what they mean and why they might have chosen it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1. The simplest definition
|
||||||
|
|
||||||
|
Both ETL and ELT describe processes for moving and transforming data.
|
||||||
|
|
||||||
|
The letters mean:
|
||||||
|
|
||||||
|
```text
|
||||||
|
E = Extract
|
||||||
|
T = Transform
|
||||||
|
L = Load
|
||||||
|
```
|
||||||
|
|
||||||
|
The difference is **the order**.
|
||||||
|
|
||||||
|
### ETL
|
||||||
|
|
||||||
|
```text
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Transform
|
||||||
|
↓
|
||||||
|
Load
|
||||||
|
```
|
||||||
|
|
||||||
|
### ELT
|
||||||
|
|
||||||
|
```text
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Load
|
||||||
|
↓
|
||||||
|
Transform
|
||||||
|
```
|
||||||
|
|
||||||
|
That tiny difference represents an important architectural decision.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2. ETL
|
||||||
|
|
||||||
|
## Extract → Transform → Load
|
||||||
|
|
||||||
|
Imagine we want data from JW Player.
|
||||||
|
|
||||||
|
We first:
|
||||||
|
|
||||||
|
### Extract
|
||||||
|
|
||||||
|
Get the data from JW Player.
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player API
|
||||||
|
↓
|
||||||
|
Raw viewing data
|
||||||
|
```
|
||||||
|
|
||||||
|
Then:
|
||||||
|
|
||||||
|
### Transform
|
||||||
|
|
||||||
|
Before putting it into our final destination, we:
|
||||||
|
|
||||||
|
- remove duplicates
|
||||||
|
|
||||||
|
- fix dates
|
||||||
|
|
||||||
|
- standardize IDs
|
||||||
|
|
||||||
|
- remove invalid records
|
||||||
|
|
||||||
|
- calculate fields
|
||||||
|
|
||||||
|
- join datasets
|
||||||
|
|
||||||
|
|
||||||
|
Then:
|
||||||
|
|
||||||
|
### Load
|
||||||
|
|
||||||
|
We put the transformed data into the destination.
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player
|
||||||
|
↓
|
||||||
|
EXTRACT
|
||||||
|
↓
|
||||||
|
Raw data
|
||||||
|
↓
|
||||||
|
TRANSFORM
|
||||||
|
↓
|
||||||
|
Clean data
|
||||||
|
↓
|
||||||
|
LOAD
|
||||||
|
↓
|
||||||
|
Data Warehouse
|
||||||
|
```
|
||||||
|
|
||||||
|
The important idea:
|
||||||
|
|
||||||
|
> **The transformation happens BEFORE the data reaches its destination.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3. Why was ETL traditionally popular?
|
||||||
|
|
||||||
|
Historically, data warehouses were:
|
||||||
|
|
||||||
|
- expensive
|
||||||
|
|
||||||
|
- highly structured
|
||||||
|
|
||||||
|
- limited in storage
|
||||||
|
|
||||||
|
- optimized for specific analytical workloads
|
||||||
|
|
||||||
|
|
||||||
|
You didn't want to dump everything into them.
|
||||||
|
|
||||||
|
You wanted to clean the data first.
|
||||||
|
|
||||||
|
Think of it like moving into a small apartment.
|
||||||
|
|
||||||
|
You don't bring everything you've accumulated over 20 years.
|
||||||
|
|
||||||
|
You:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Pack
|
||||||
|
↓
|
||||||
|
Sort
|
||||||
|
↓
|
||||||
|
Throw things away
|
||||||
|
↓
|
||||||
|
Move
|
||||||
|
```
|
||||||
|
|
||||||
|
That's ETL.
|
||||||
|
|
||||||
|
You decide what is useful **before loading it into the destination**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4. ELT
|
||||||
|
|
||||||
|
Modern cloud platforms changed things.
|
||||||
|
|
||||||
|
Storage became much cheaper.
|
||||||
|
|
||||||
|
Compute became much more scalable.
|
||||||
|
|
||||||
|
So organizations started saying:
|
||||||
|
|
||||||
|
> Why don't we load the raw data first and transform it afterward?
|
||||||
|
|
||||||
|
That's:
|
||||||
|
|
||||||
|
## Extract → Load → Transform
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player
|
||||||
|
↓
|
||||||
|
EXTRACT
|
||||||
|
↓
|
||||||
|
LOAD
|
||||||
|
↓
|
||||||
|
Data Lake / Warehouse
|
||||||
|
↓
|
||||||
|
TRANSFORM
|
||||||
|
↓
|
||||||
|
Cleaned data
|
||||||
|
```
|
||||||
|
|
||||||
|
The key difference:
|
||||||
|
|
||||||
|
> **We preserve the raw data in the destination before transforming it.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5. Your media-company example
|
||||||
|
|
||||||
|
Imagine you receive **100 million JW Player events**.
|
||||||
|
|
||||||
|
With traditional ETL:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player
|
||||||
|
↓
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Transformation Server
|
||||||
|
↓
|
||||||
|
Clean / Aggregate
|
||||||
|
↓
|
||||||
|
Data Warehouse
|
||||||
|
```
|
||||||
|
|
||||||
|
You might only load the processed results.
|
||||||
|
|
||||||
|
Perhaps:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Show A → 50,000 views
|
||||||
|
Show B → 25,000 views
|
||||||
|
```
|
||||||
|
|
||||||
|
But what happens six months later when someone asks:
|
||||||
|
|
||||||
|
> "Can we calculate average viewing duration differently?"
|
||||||
|
|
||||||
|
Potential problem.
|
||||||
|
|
||||||
|
Maybe the original detailed data wasn't preserved.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
With ELT:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player
|
||||||
|
↓
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Data Lake
|
||||||
|
↓
|
||||||
|
RAW DATA STORED
|
||||||
|
↓
|
||||||
|
Transform
|
||||||
|
↓
|
||||||
|
Silver
|
||||||
|
↓
|
||||||
|
Transform
|
||||||
|
↓
|
||||||
|
Gold
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you still have the original events.
|
||||||
|
|
||||||
|
You can go back.
|
||||||
|
|
||||||
|
Reprocess.
|
||||||
|
|
||||||
|
Create new metrics.
|
||||||
|
|
||||||
|
Investigate problems.
|
||||||
|
|
||||||
|
That's a major advantage.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6. Connecting this to Medallion Architecture
|
||||||
|
|
||||||
|
This is where everything we've learned starts connecting.
|
||||||
|
|
||||||
|
A modern architecture might look like:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SOURCE SYSTEMS
|
||||||
|
|
||||||
|
GA4
|
||||||
|
JW Player
|
||||||
|
CRM
|
||||||
|
Finance
|
||||||
|
OTT
|
||||||
|
Apps
|
||||||
|
|
||||||
|
│
|
||||||
|
│ EXTRACT
|
||||||
|
▼
|
||||||
|
|
||||||
|
┌─────────────────┐
|
||||||
|
│ BRONZE │
|
||||||
|
│ │
|
||||||
|
│ RAW DATA │
|
||||||
|
└────────┬────────┘
|
||||||
|
│
|
||||||
|
│ TRANSFORM
|
||||||
|
▼
|
||||||
|
┌─────────────────┐
|
||||||
|
│ SILVER │
|
||||||
|
│ │
|
||||||
|
│ CLEAN DATA │
|
||||||
|
└────────┬────────┘
|
||||||
|
│
|
||||||
|
│ TRANSFORM
|
||||||
|
▼
|
||||||
|
┌─────────────────┐
|
||||||
|
│ GOLD │
|
||||||
|
│ │
|
||||||
|
│ BUSINESS READY │
|
||||||
|
└────────┬────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
This architecture is conceptually closer to **ELT**.
|
||||||
|
|
||||||
|
Why?
|
||||||
|
|
||||||
|
Because we:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Load raw data
|
||||||
|
↓
|
||||||
|
Transform afterward
|
||||||
|
```
|
||||||
|
|
||||||
|
The transformations create increasingly refined datasets.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 7. ETL vs ELT
|
||||||
|
|
||||||
|
Here's your executive-level comparison:
|
||||||
|
|
||||||
|
||ETL|ELT|
|
||||||
|
|---|---|---|
|
||||||
|
|Order|Extract → Transform → Load|Extract → Load → Transform|
|
||||||
|
|Raw data stored first|Not necessarily|Usually|
|
||||||
|
|Transformation location|Before destination|In/around destination platform|
|
||||||
|
|Flexibility|Lower|Higher|
|
||||||
|
|Reprocessing|Potentially harder|Easier if raw data retained|
|
||||||
|
|Traditional warehouses|Common|Less traditional|
|
||||||
|
|Modern cloud data platforms|Used|Very common|
|
||||||
|
|
||||||
|
Neither is automatically better.
|
||||||
|
|
||||||
|
Architecture depends on:
|
||||||
|
|
||||||
|
- data volume
|
||||||
|
|
||||||
|
- cost
|
||||||
|
|
||||||
|
- security
|
||||||
|
|
||||||
|
- privacy
|
||||||
|
|
||||||
|
- latency
|
||||||
|
|
||||||
|
- technology
|
||||||
|
|
||||||
|
- business requirements
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 8. A subtle but important point
|
||||||
|
|
||||||
|
People often use the word:
|
||||||
|
|
||||||
|
> **Pipeline**
|
||||||
|
|
||||||
|
A data pipeline is simply an automated process that moves and/or transforms data.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player API
|
||||||
|
↓
|
||||||
|
Pipeline
|
||||||
|
↓
|
||||||
|
Bronze
|
||||||
|
```
|
||||||
|
|
||||||
|
Another pipeline might do:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Bronze
|
||||||
|
↓
|
||||||
|
Pipeline
|
||||||
|
↓
|
||||||
|
Silver
|
||||||
|
```
|
||||||
|
|
||||||
|
Another:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Silver
|
||||||
|
↓
|
||||||
|
Pipeline
|
||||||
|
↓
|
||||||
|
Gold
|
||||||
|
```
|
||||||
|
|
||||||
|
So your entire data architecture might contain dozens or hundreds of pipelines.
|
||||||
|
|
||||||
|
Some run:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Every 5 minutes
|
||||||
|
```
|
||||||
|
|
||||||
|
Some:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Hourly
|
||||||
|
```
|
||||||
|
|
||||||
|
Some:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Nightly
|
||||||
|
```
|
||||||
|
|
||||||
|
Some:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Real-time
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 9. Batch vs Streaming
|
||||||
|
|
||||||
|
This is related, but it's a separate concept we'll study later.
|
||||||
|
|
||||||
|
For now, just understand:
|
||||||
|
|
||||||
|
### Batch
|
||||||
|
|
||||||
|
Data moves periodically.
|
||||||
|
|
||||||
|
```text
|
||||||
|
Every night at 2 AM:
|
||||||
|
|
||||||
|
Yesterday's JW Player data
|
||||||
|
↓
|
||||||
|
Pipeline
|
||||||
|
↓
|
||||||
|
Data Lake
|
||||||
|
```
|
||||||
|
|
||||||
|
### Streaming
|
||||||
|
|
||||||
|
Data continuously flows.
|
||||||
|
|
||||||
|
```text
|
||||||
|
User watches video
|
||||||
|
↓
|
||||||
|
Event
|
||||||
|
↓
|
||||||
|
Pipeline
|
||||||
|
↓
|
||||||
|
Data Platform
|
||||||
|
```
|
||||||
|
|
||||||
|
Think:
|
||||||
|
|
||||||
|
> **Batch = packages of data periodically.**
|
||||||
|
|
||||||
|
> **Streaming = continuous flow of events.**
|
||||||
|
|
||||||
|
This distinction can become important for a media company.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 10. Where does Azure Synapse fit?
|
||||||
|
|
||||||
|
Remember Lesson 2:
|
||||||
|
|
||||||
|
Synapse is a **platform**.
|
||||||
|
|
||||||
|
A company could potentially use Synapse capabilities to orchestrate pipelines that:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Extract
|
||||||
|
↓
|
||||||
|
Load
|
||||||
|
↓
|
||||||
|
Transform
|
||||||
|
```
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player
|
||||||
|
↓
|
||||||
|
Synapse Pipeline
|
||||||
|
↓
|
||||||
|
Azure Data Lake
|
||||||
|
↓
|
||||||
|
Bronze
|
||||||
|
↓
|
||||||
|
Transformation
|
||||||
|
↓
|
||||||
|
Silver
|
||||||
|
↓
|
||||||
|
Transformation
|
||||||
|
↓
|
||||||
|
Gold
|
||||||
|
↓
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
Again, your company's actual architecture might differ.
|
||||||
|
|
||||||
|
When you're back from vacation, that's something worth mapping.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 11. CTO perspective
|
||||||
|
|
||||||
|
Imagine an architect says:
|
||||||
|
|
||||||
|
> "We're moving from ETL to ELT."
|
||||||
|
|
||||||
|
Don't just respond:
|
||||||
|
|
||||||
|
> "Okay."
|
||||||
|
|
||||||
|
You want to understand **why**.
|
||||||
|
|
||||||
|
You could ask:
|
||||||
|
|
||||||
|
> "What's driving the change?"
|
||||||
|
|
||||||
|
Then:
|
||||||
|
|
||||||
|
> "Are we preserving all raw source data?"
|
||||||
|
|
||||||
|
> "Where are transformations executed?"
|
||||||
|
|
||||||
|
> "What's the impact on compute costs?"
|
||||||
|
|
||||||
|
> "How are transformations tested?"
|
||||||
|
|
||||||
|
> "How do we handle schema changes from source systems?"
|
||||||
|
|
||||||
|
> "Can we reprocess historical data if business rules change?"
|
||||||
|
|
||||||
|
These questions reveal the architectural consequences.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 12. Meeting scenario
|
||||||
|
|
||||||
|
Imagine your analyst says:
|
||||||
|
|
||||||
|
> "We need to change the way we calculate total viewing hours, but we'd have to reload everything."
|
||||||
|
|
||||||
|
Instead of immediately discussing implementation, you could ask:
|
||||||
|
|
||||||
|
> **"Do we still have the original raw viewing events in our Bronze layer?"**
|
||||||
|
|
||||||
|
If yes:
|
||||||
|
|
||||||
|
> **"Then can we reprocess the affected data through Silver and regenerate the Gold dataset using the new business rule?"**
|
||||||
|
|
||||||
|
Now you're thinking architecturally.
|
||||||
|
|
||||||
|
You understand:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Bronze
|
||||||
|
Original truth
|
||||||
|
↓
|
||||||
|
Silver
|
||||||
|
Clean data
|
||||||
|
↓
|
||||||
|
Gold
|
||||||
|
Business interpretation
|
||||||
|
```
|
||||||
|
|
||||||
|
If the **business rule** changes, you may not need new source data.
|
||||||
|
|
||||||
|
You may need to **reprocess existing data**.
|
||||||
|
|
||||||
|
That's one reason retaining raw data can be valuable.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 13. Another CTO-level issue: Garbage In, Garbage Out
|
||||||
|
|
||||||
|
ELT has a potential downside.
|
||||||
|
|
||||||
|
Because storing raw data is easy, organizations sometimes end up with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Data Lake
|
||||||
|
↓
|
||||||
|
More Data
|
||||||
|
↓
|
||||||
|
More Data
|
||||||
|
↓
|
||||||
|
More Data
|
||||||
|
↓
|
||||||
|
Nobody knows what anything means
|
||||||
|
```
|
||||||
|
|
||||||
|
This is sometimes jokingly called a:
|
||||||
|
|
||||||
|
> **Data Swamp**
|
||||||
|
|
||||||
|
A Data Lake without proper:
|
||||||
|
|
||||||
|
- governance
|
||||||
|
|
||||||
|
- metadata
|
||||||
|
|
||||||
|
- ownership
|
||||||
|
|
||||||
|
- documentation
|
||||||
|
|
||||||
|
- lineage
|
||||||
|
|
||||||
|
- quality controls
|
||||||
|
|
||||||
|
|
||||||
|
can become difficult to use.
|
||||||
|
|
||||||
|
This connects directly to your role in **Data Governance**.
|
||||||
|
|
||||||
|
The technology doesn't create trustworthy data.
|
||||||
|
|
||||||
|
Governance does.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 14. Your executive communication exercise
|
||||||
|
|
||||||
|
Someone says in a meeting:
|
||||||
|
|
||||||
|
> "We should move everything to ELT because it's the modern approach."
|
||||||
|
|
||||||
|
Your response could be:
|
||||||
|
|
||||||
|
> **"Before we decide on ETL versus ELT, I'd like to understand the problem we're solving. If the goal is to preserve raw data and give us more flexibility to reprocess it as business requirements change, ELT may make sense. But we should also understand the impact on storage, compute, governance and data quality before changing the architecture."**
|
||||||
|
|
||||||
|
Notice the pattern again:
|
||||||
|
|
||||||
|
**Clarify the problem → acknowledge the option → identify tradeoffs → guide the decision.**
|
||||||
|
|
||||||
|
You don't need to be the data engineer.
|
||||||
|
|
||||||
|
You need to make sure the organization is solving the right problem.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Your Cheat Sheet
|
||||||
|
|
||||||
|
Remember this:
|
||||||
|
|
||||||
|
> **ETL:** Clean it, then load it.
|
||||||
|
|
||||||
|
> **ELT:** Load it, then clean it.
|
||||||
|
|
||||||
|
> **Pipeline:** Automated movement/transformation of data.
|
||||||
|
|
||||||
|
> **Batch:** Move data periodically.
|
||||||
|
|
||||||
|
> **Streaming:** Move data continuously.
|
||||||
|
|
||||||
|
And the connection between our first three lessons:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SOURCE SYSTEMS
|
||||||
|
│
|
||||||
|
│
|
||||||
|
Extract / Load
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
🥉 BRONZE
|
||||||
|
Raw Data
|
||||||
|
│
|
||||||
|
Transform
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
🥈 SILVER
|
||||||
|
Clean Data
|
||||||
|
│
|
||||||
|
Transform
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
🥇 GOLD
|
||||||
|
Business Data
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
That's the mental model I want you to keep.
|
||||||
|
|
||||||
|
## Add to Obsidian
|
||||||
|
|
||||||
|
Create:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Knowledge
|
||||||
|
└── CTO Academy
|
||||||
|
└── Data
|
||||||
|
└── ETL vs ELT.md
|
||||||
|
```
|
||||||
|
|
||||||
|
At the bottom:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
## Related Concepts
|
||||||
|
|
||||||
|
- [[Medallion Architecture]]
|
||||||
|
- [[Data Lake]]
|
||||||
|
- [[Data Warehouse]]
|
||||||
|
- [[Lakehouse]]
|
||||||
|
- [[Azure Synapse]]
|
||||||
|
- [[Data Pipeline]]
|
||||||
|
- [[Batch Processing]]
|
||||||
|
- [[Stream Processing]]
|
||||||
|
- [[Data Governance]]
|
||||||
|
- [[Data Lineage]]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Next: Lesson 4
|
||||||
|
|
||||||
|
I think we're ready to move **inside the Gold layer**.
|
||||||
|
|
||||||
|
**Fact Tables & Dimension Tables.**
|
||||||
|
|
||||||
|
You've encountered these concepts before when we were playing with Metabase and discussing grain/granularité, so Lesson 4 should connect something you already partially understand with the architecture we're building now:
|
||||||
|
|
||||||
|
**Source → ELT → Bronze → Silver → Gold → Fact/Dimensions → Power BI.**
|
||||||
|
|
||||||
|
That's when this whole data architecture should really start clicking together.
|
||||||
@@ -0,0 +1,405 @@
|
|||||||
|
## Bronze, Silver & Gold
|
||||||
|
|
||||||
|
**Time: ~10 minutes**
|
||||||
|
|
||||||
|
### 1. Definition
|
||||||
|
|
||||||
|
**Medallion Architecture** is a way of organizing data into layers based on how processed and trustworthy it is.
|
||||||
|
|
||||||
|
The three common layers are:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DATA SOURCES
|
||||||
|
↓
|
||||||
|
┌──────────────┐
|
||||||
|
│ BRONZE │ Raw data
|
||||||
|
└──────────────┘
|
||||||
|
↓
|
||||||
|
┌──────────────┐
|
||||||
|
│ SILVER │ Cleaned & standardized
|
||||||
|
└──────────────┘
|
||||||
|
↓
|
||||||
|
┌──────────────┐
|
||||||
|
│ GOLD │ Business-ready
|
||||||
|
└──────────────┘
|
||||||
|
↓
|
||||||
|
Power BI / Analytics / KPIs / Applications
|
||||||
|
```
|
||||||
|
|
||||||
|
The fundamental idea is:
|
||||||
|
|
||||||
|
> **Raw → Clean → Business-ready**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2. Why does this exist?
|
||||||
|
|
||||||
|
Imagine your company collects data from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GA4
|
||||||
|
JW Player
|
||||||
|
Cable/linear TV
|
||||||
|
Advertising
|
||||||
|
CRM
|
||||||
|
Finance
|
||||||
|
HR
|
||||||
|
CMS
|
||||||
|
Mobile apps
|
||||||
|
OTT apps
|
||||||
|
```
|
||||||
|
|
||||||
|
Every system has different formats, naming conventions and levels of quality.
|
||||||
|
|
||||||
|
You don't want your Power BI analysts independently cleaning and interpreting all of that data every time they create a report.
|
||||||
|
|
||||||
|
Instead, you progressively transform the data.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🥉 Bronze: "What did we receive?"
|
||||||
|
|
||||||
|
Bronze is the **raw data**.
|
||||||
|
|
||||||
|
Ideally, you preserve the data approximately as it arrived from the source.
|
||||||
|
|
||||||
|
Example from JW Player:
|
||||||
|
|
||||||
|
```text
|
||||||
|
user_id video_id play_time timestamp
|
||||||
|
82372 VID123 134 2026-07-18 12:34
|
||||||
|
82373 VID555 NULL 2026-07-18 12:35
|
||||||
|
82372 VID123 134 2026-07-18 12:34
|
||||||
|
```
|
||||||
|
|
||||||
|
Notice:
|
||||||
|
|
||||||
|
- NULL values
|
||||||
|
|
||||||
|
- duplicates
|
||||||
|
|
||||||
|
- potentially incorrect data
|
||||||
|
|
||||||
|
|
||||||
|
That's okay.
|
||||||
|
|
||||||
|
Bronze isn't necessarily supposed to be beautiful.
|
||||||
|
|
||||||
|
### Why keep it?
|
||||||
|
|
||||||
|
Because if your transformation is wrong, you can go back to the original data.
|
||||||
|
|
||||||
|
Think:
|
||||||
|
|
||||||
|
> **Bronze = source of historical truth about what we received.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🥈 Silver: "What data can we trust technically?"
|
||||||
|
|
||||||
|
Now we clean it.
|
||||||
|
|
||||||
|
We might:
|
||||||
|
|
||||||
|
- Remove duplicates
|
||||||
|
|
||||||
|
- Standardize dates
|
||||||
|
|
||||||
|
- Handle NULL values
|
||||||
|
|
||||||
|
- Validate IDs
|
||||||
|
|
||||||
|
- Standardize country codes
|
||||||
|
|
||||||
|
- Join related datasets
|
||||||
|
|
||||||
|
- Correct data types
|
||||||
|
|
||||||
|
|
||||||
|
Our data becomes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
user_id video_id watch_seconds date
|
||||||
|
82372 VID123 134 2026-07-18
|
||||||
|
```
|
||||||
|
|
||||||
|
Now it's consistent.
|
||||||
|
|
||||||
|
But here's an important distinction:
|
||||||
|
|
||||||
|
**Silver doesn't necessarily understand the business.**
|
||||||
|
|
||||||
|
It understands the **data**.
|
||||||
|
|
||||||
|
Think:
|
||||||
|
|
||||||
|
> **Silver = clean and standardized data.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🥇 Gold: "What does the business need?"
|
||||||
|
|
||||||
|
This is where your original question comes in.
|
||||||
|
|
||||||
|
Gold data is designed for **business consumption**.
|
||||||
|
|
||||||
|
Instead of millions of video events, you might have:
|
||||||
|
|
||||||
|
```text
|
||||||
|
content viewers watch_hours
|
||||||
|
Show A 152,000 47,500
|
||||||
|
Show B 89,000 31,200
|
||||||
|
Show C 67,000 19,800
|
||||||
|
```
|
||||||
|
|
||||||
|
Or:
|
||||||
|
|
||||||
|
```text
|
||||||
|
month streaming_hours unique_viewers
|
||||||
|
January 1,500,000 320,000
|
||||||
|
February 1,700,000 350,000
|
||||||
|
March 1,850,000 380,000
|
||||||
|
```
|
||||||
|
|
||||||
|
Now Power BI can consume this directly.
|
||||||
|
|
||||||
|
Think:
|
||||||
|
|
||||||
|
> **Gold = data organized around business questions.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3. Your TV-company example
|
||||||
|
|
||||||
|
Let's imagine your CEO asks:
|
||||||
|
|
||||||
|
> "How many people watched our content last month across all digital platforms?"
|
||||||
|
|
||||||
|
The architecture could look like this:
|
||||||
|
|
||||||
|
```text
|
||||||
|
JW Player ──────────┐
|
||||||
|
│
|
||||||
|
GA4 ────────────────┤
|
||||||
|
│
|
||||||
|
Mobile App ─────────┤
|
||||||
|
↓
|
||||||
|
BRONZE
|
||||||
|
Raw source data
|
||||||
|
↓
|
||||||
|
SILVER
|
||||||
|
Clean IDs / Remove duplicates
|
||||||
|
Standardize timestamps
|
||||||
|
Match content identifiers
|
||||||
|
↓
|
||||||
|
GOLD
|
||||||
|
Monthly Content Audience
|
||||||
|
↓
|
||||||
|
Power BI
|
||||||
|
↓
|
||||||
|
CEO
|
||||||
|
```
|
||||||
|
|
||||||
|
Your CEO should never have to understand Bronze.
|
||||||
|
|
||||||
|
Your Power BI developer ideally shouldn't repeatedly rebuild Silver transformations.
|
||||||
|
|
||||||
|
The Gold layer should provide trusted business-ready datasets.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4. The CTO perspective
|
||||||
|
|
||||||
|
Here's where **your role** becomes important.
|
||||||
|
|
||||||
|
You don't necessarily need to know how your data engineer writes the transformation.
|
||||||
|
|
||||||
|
You need to ask questions like:
|
||||||
|
|
||||||
|
> **Who owns the Gold layer?**
|
||||||
|
|
||||||
|
> **Who defines the business rules?**
|
||||||
|
|
||||||
|
> **How do we know these KPIs are correct?**
|
||||||
|
|
||||||
|
> **Can we trace a Gold KPI back to its original source?**
|
||||||
|
|
||||||
|
> **What happens when the source changes?**
|
||||||
|
|
||||||
|
> **Are Power BI analysts calculating KPIs independently, or consuming governed metrics?**
|
||||||
|
|
||||||
|
That last question is particularly relevant to your Data Governance responsibilities.
|
||||||
|
|
||||||
|
Imagine:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Power BI Analyst A
|
||||||
|
"Active User = logged-in user"
|
||||||
|
|
||||||
|
Power BI Analyst B
|
||||||
|
"Active User = user with a session"
|
||||||
|
|
||||||
|
Marketing
|
||||||
|
"Active User = user who watched content"
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you have three numbers.
|
||||||
|
|
||||||
|
A proper Gold layer might establish:
|
||||||
|
|
||||||
|
```text
|
||||||
|
dim_user
|
||||||
|
fact_viewing
|
||||||
|
fact_sessions
|
||||||
|
fact_subscriptions
|
||||||
|
```
|
||||||
|
|
||||||
|
and governed KPI definitions determine how those datasets are interpreted.
|
||||||
|
|
||||||
|
We'll get into **fact and dimension tables** in another lesson.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5. Something important: Medallion ≠ Star Schema
|
||||||
|
|
||||||
|
You will eventually hear both terms.
|
||||||
|
|
||||||
|
They're related, but they're answering different questions.
|
||||||
|
|
||||||
|
**Medallion architecture asks:**
|
||||||
|
|
||||||
|
> How processed is the data?
|
||||||
|
|
||||||
|
```text
|
||||||
|
Bronze → Silver → Gold
|
||||||
|
```
|
||||||
|
|
||||||
|
**Star schema asks:**
|
||||||
|
|
||||||
|
> How should analytical data be modeled?
|
||||||
|
|
||||||
|
```text
|
||||||
|
Dimension
|
||||||
|
|
|
||||||
|
Dimension - Fact - Dimension
|
||||||
|
|
|
||||||
|
Dimension
|
||||||
|
```
|
||||||
|
|
||||||
|
A **Gold-layer dataset could be modeled as a star schema**.
|
||||||
|
|
||||||
|
That's why these concepts often appear in the same conversation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6. Common mistake
|
||||||
|
|
||||||
|
A company sometimes says:
|
||||||
|
|
||||||
|
> "We have a Gold layer."
|
||||||
|
|
||||||
|
But what they really have is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Raw Data
|
||||||
|
↓
|
||||||
|
A bunch of SQL
|
||||||
|
↓
|
||||||
|
Power BI
|
||||||
|
```
|
||||||
|
|
||||||
|
The word "Gold" doesn't automatically mean the data is trustworthy.
|
||||||
|
|
||||||
|
A CTO should care about:
|
||||||
|
|
||||||
|
- Ownership
|
||||||
|
|
||||||
|
- Documentation
|
||||||
|
|
||||||
|
- Data quality
|
||||||
|
|
||||||
|
- Business definitions
|
||||||
|
|
||||||
|
- Lineage
|
||||||
|
|
||||||
|
- Testing
|
||||||
|
|
||||||
|
|
||||||
|
We'll cover **Data Lineage** separately because it's another term you're very likely to encounter.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 7. Meeting scenario
|
||||||
|
|
||||||
|
Imagine your data analyst says:
|
||||||
|
|
||||||
|
> "We should create this KPI in the Gold layer."
|
||||||
|
|
||||||
|
You could respond:
|
||||||
|
|
||||||
|
> **"That makes sense. Before we do that, I want to make sure we agree on the business definition of the KPI and who owns that definition. Once that's established, we can expose the governed metric through the Gold layer so Power BI and other consumers use the same calculation."**
|
||||||
|
|
||||||
|
That's a strong manager/CTO answer.
|
||||||
|
|
||||||
|
You're not pretending to know how they'll implement it.
|
||||||
|
|
||||||
|
You're focusing on **governance, consistency and business value**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 8. Your cheat sheet
|
||||||
|
|
||||||
|
Remember only this:
|
||||||
|
|
||||||
|
> 🥉 **Bronze:** What did we receive?
|
||||||
|
>
|
||||||
|
> 🥈 **Silver:** What data is technically clean and trustworthy?
|
||||||
|
>
|
||||||
|
> 🥇 **Gold:** What does the business need?
|
||||||
|
>
|
||||||
|
> **Medallion architecture:** Raw → Clean → Business-ready
|
||||||
|
>
|
||||||
|
> **Your CTO question:** Who owns it, who defines it, and can we trust it?
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Add this to Obsidian
|
||||||
|
|
||||||
|
I'd create:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Knowledge
|
||||||
|
└── CTO Academy
|
||||||
|
└── Data
|
||||||
|
└── Medallion Architecture.md
|
||||||
|
```
|
||||||
|
|
||||||
|
At the bottom:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
## Related Concepts
|
||||||
|
|
||||||
|
- [[Data Warehouse]]
|
||||||
|
- [[Data Lake vs Data Warehouse vs Lakehouse]]
|
||||||
|
- [[Lakehouse]]
|
||||||
|
- [[ETL]]
|
||||||
|
- [[ELT]]
|
||||||
|
- [[Star Schema]]
|
||||||
|
- [[Fact Table]]
|
||||||
|
- [[Dimension Table]]
|
||||||
|
- [[Data Lineage]]
|
||||||
|
- [[Data Governance]]
|
||||||
|
- [[Semantic Layer]]
|
||||||
|
```
|
||||||
|
|
||||||
|
### One question for you before we consider today's lesson complete:
|
||||||
|
|
||||||
|
Your Power BI team currently receives data from multiple sources and creates reports.
|
||||||
|
|
||||||
|
**Where do you think your current environment sits today?**
|
||||||
|
|
||||||
|
Do you think you have something resembling Bronze → Silver → Gold, or are your analysts mostly taking source data and doing the cleaning/business logic directly inside Power BI?
|
||||||
|
|
||||||
|
My answer:
|
||||||
|
|
||||||
|
We definitely have a bronze, silver and gold. Power Bi is only used to create the reports so it consumes the gold layer. I do know there's azure synapse, data lake, data warehouses, so all of these should be for bronze and silver layers.
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
## Definition
|
||||||
|
|
||||||
|
What is it?
|
||||||
|
|
||||||
|
## Simple Mental Model
|
||||||
|
|
||||||
|
Explain it in one or two sentences.
|
||||||
|
|
||||||
|
## How It Works
|
||||||
|
|
||||||
|
Explain the concept.
|
||||||
|
|
||||||
|
## Example
|
||||||
|
|
||||||
|
Provide a practical example.
|
||||||
|
|
||||||
|
## How It Fits Into the Bigger Picture
|
||||||
|
|
||||||
|
Connect it to concepts I already know.
|
||||||
|
|
||||||
|
## My Company / Real-World Context
|
||||||
|
|
||||||
|
How could this apply to my environment?
|
||||||
|
|
||||||
|
## CTO Perspective
|
||||||
|
|
||||||
|
What does a CTO or technology leader need to understand about this?
|
||||||
|
|
||||||
|
### Questions to Ask
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Meeting Scenario
|
||||||
|
|
||||||
|
**Situation:**
|
||||||
|
|
||||||
|
**Possible response:**
|
||||||
|
|
||||||
|
>
|
||||||
|
|
||||||
|
## Key Takeaways
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Related Concepts
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
**Date:** {{date}}
|
||||||
|
**Status:** 🟡 Proposed
|
||||||
|
**Decision Owner:**
|
||||||
|
**People Involved:**
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
What situation led to this decision?
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
What problem are we trying to solve?
|
||||||
|
|
||||||
|
## Options Considered
|
||||||
|
|
||||||
|
### Option 1
|
||||||
|
|
||||||
|
**Description:**
|
||||||
|
|
||||||
|
**Pros:**
|
||||||
|
-
|
||||||
|
|
||||||
|
**Cons:**
|
||||||
|
-
|
||||||
|
|
||||||
|
### Option 2
|
||||||
|
|
||||||
|
**Description:**
|
||||||
|
|
||||||
|
**Pros:**
|
||||||
|
-
|
||||||
|
|
||||||
|
**Cons:**
|
||||||
|
-
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
What did we decide?
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
Why was this option chosen?
|
||||||
|
|
||||||
|
## Risks / Trade-offs
|
||||||
|
|
||||||
|
What are we accepting by making this decision?
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Outcome
|
||||||
|
|
||||||
|
_To be completed later._
|
||||||
|
|
||||||
|
What actually happened after the decision?
|
||||||
|
|
||||||
|
## Lessons Learned
|
||||||
|
|
||||||
|
What would I do differently next time?
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
## Role
|
||||||
|
|
||||||
|
**Position:**
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
## Current Objectives
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Current Projects
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## 1:1 Notes
|
||||||
|
|
||||||
|
### {{date}}
|
||||||
|
|
||||||
|
**Topics discussed:**
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
**Employee feedback / concerns:**
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
**My feedback:**
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
**Agreed actions:**
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Performance Observations
|
||||||
|
|
||||||
|
### {{date}}
|
||||||
|
|
||||||
|
**Situation:**
|
||||||
|
|
||||||
|
**Expected outcome:**
|
||||||
|
|
||||||
|
**Actual outcome:**
|
||||||
|
|
||||||
|
**Impact:**
|
||||||
|
|
||||||
|
**Follow-up:**
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Strengths
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
### Areas for Development
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
### Support / Coaching Provided
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Commitments
|
||||||
|
|
||||||
|
| Commitment | Owner | Due | Status |
|
||||||
|
|---|---|---|---|
|
||||||
|
| | | | |
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
**Date:** {{date}}
|
||||||
|
**Status:** 💡 Idea
|
||||||
|
|
||||||
|
## The Idea
|
||||||
|
|
||||||
|
Describe the idea simply.
|
||||||
|
|
||||||
|
## Problem / Opportunity
|
||||||
|
|
||||||
|
What problem does this solve or what opportunity does it create?
|
||||||
|
|
||||||
|
## Why It Matters
|
||||||
|
|
||||||
|
What is the potential business value?
|
||||||
|
|
||||||
|
## Who Benefits?
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Possible Approach
|
||||||
|
|
||||||
|
How could this work?
|
||||||
|
|
||||||
|
## Effort
|
||||||
|
|
||||||
|
**Estimated complexity:** Low / Medium / High
|
||||||
|
|
||||||
|
## Risks / Questions
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Next Step
|
||||||
|
|
||||||
|
What is the smallest action needed to validate this idea?
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# {{date}}
|
||||||
|
|
||||||
|
## What Happened
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## People / Management
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## What I Learned
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## What I Could Have Done Better
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Follow-up
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
**Date:** {{date}}
|
||||||
|
**Attendees:**
|
||||||
|
**Purpose:**
|
||||||
|
|
||||||
|
## Desired Outcome
|
||||||
|
|
||||||
|
What needs to be achieved or decided in this meeting?
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Brief background needed to understand the discussion.
|
||||||
|
|
||||||
|
## Agenda
|
||||||
|
|
||||||
|
1.
|
||||||
|
2.
|
||||||
|
3.
|
||||||
|
|
||||||
|
## Key Discussion Points
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Decisions Made
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Action Items
|
||||||
|
|
||||||
|
- [ ] Action — **Owner:** — **Due:**
|
||||||
|
- [ ] Action — **Owner:** — **Due:**
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Follow-up
|
||||||
|
|
||||||
|
What needs to happen next?
|
||||||
|
|
||||||
|
## My Notes
|
||||||
|
|
||||||
|
Anything I noticed about the discussion, dynamics, risks, or opportunities.
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# {{title}}
|
||||||
|
|
||||||
|
**Status:** 🟢 Active
|
||||||
|
**Owner:**
|
||||||
|
**Start Date:** {{date}}
|
||||||
|
**Target Date:**
|
||||||
|
**Stakeholders:**
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
What are we trying to accomplish?
|
||||||
|
|
||||||
|
## Business Value
|
||||||
|
|
||||||
|
Why are we doing this?
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
How will we know the project succeeded?
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In Scope
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
### Out of Scope
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Deliverables
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Milestones
|
||||||
|
|
||||||
|
| Milestone | Owner | Target | Status |
|
||||||
|
|---|---|---|---|
|
||||||
|
| | | | |
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
|
||||||
|
| Risk | Impact | Probability | Mitigation |
|
||||||
|
|---|---|---|---|
|
||||||
|
| | | | |
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
- [[]]
|
||||||
|
|
||||||
|
## Current Status
|
||||||
|
|
||||||
|
**Overall:** 🟢 On Track / 🟡 At Risk / 🔴 Blocked
|
||||||
|
|
||||||
|
### Completed
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
### In Progress
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
### Blockers
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Next Actions
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Lessons Learned
|
||||||
|
|
||||||
|
_To be completed during and after the project._
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
|
||||||
|
|
||||||
|
**Date:** 2026-07-18
|
||||||
|
**Status:** 💡 Idea
|
||||||
|
|
||||||
|
## The Idea
|
||||||
|
|
||||||
|
Idea is to have a rotation of 2 weeks where one person is in charge of the team.
|
||||||
|
Would it work ? what does it mean to the team ? What value to gain out of it ?
|
||||||
|
|
||||||
|
|
||||||
|
## Problem / Opportunity
|
||||||
|
|
||||||
|
What problem does this solve or what opportunity does it create?
|
||||||
|
|
||||||
|
## Why It Matters
|
||||||
|
|
||||||
|
What is the potential business value?
|
||||||
|
|
||||||
|
## Who Benefits?
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Possible Approach
|
||||||
|
|
||||||
|
How could this work?
|
||||||
|
|
||||||
|
## Effort
|
||||||
|
|
||||||
|
**Estimated complexity:** Low / Medium / High
|
||||||
|
|
||||||
|
## Risks / Questions
|
||||||
|
|
||||||
|
-
|
||||||
|
-
|
||||||
|
|
||||||
|
## Next Step
|
||||||
|
|
||||||
|
What is the smallest action needed to validate this idea?
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [[]]
|
||||||
+3
-3
@@ -19,6 +19,6 @@
|
|||||||
- [ ] Finish High Output Management
|
- [ ] Finish High Output Management
|
||||||
|
|
||||||
## Quick Links
|
## Quick Links
|
||||||
- [[50 Principles/]]
|
- [[50 Principles/ | Principles]]
|
||||||
- [[10 Knowledge/CTO Academy/]]
|
- [[10 Knowledge/CTO Academy/ | CTO Academy]]
|
||||||
- [[10 Knowledge/Executive Playbook/]]
|
- [[10 Knowledge/Executive Playbook/ | Executive Playbook]]
|
||||||
|
|||||||
Reference in New Issue
Block a user