From cf99ac6bd6c0c84a7e523ab868e32f51db7ee2c0 Mon Sep 17 00:00:00 2001 From: modji Date: Mon, 20 Jul 2026 21:03:03 +0200 Subject: [PATCH 1/6] vault backup: 2026-07-20 21:03:03 --- .obsidian/workspace.json | 14 +- 10 Knowledge/CTO Academy/Data/Data Lineage.md | 673 ++++++++++++++++++ .../CTO Academy/Data/Quizzes/.gitkeep | 0 3 files changed, 683 insertions(+), 4 deletions(-) create mode 100644 10 Knowledge/CTO Academy/Data/Data Lineage.md create mode 100644 10 Knowledge/CTO Academy/Data/Quizzes/.gitkeep diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index d5c799e..07b524d 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -11,10 +11,14 @@ "id": "579a503273b0c5ec", "type": "leaf", "state": { - "type": "empty", - "state": {}, + "type": "markdown", + "state": { + "file": "10 Knowledge/CTO Academy/Data/Data Lineage.md", + "mode": "source", + "source": false + }, "icon": "lucide-file", - "title": "New tab" + "title": "Data Lineage" } } ] @@ -192,8 +196,10 @@ "omnisearch:Omnisearch": false } }, - "active": "579a503273b0c5ec", + "active": "92c89fdf4eb1874e", "lastOpenFiles": [ + "10 Knowledge/CTO Academy/Data/Quizzes", + "10 Knowledge/CTO Academy/Data/Data Lineage.md", "10 Knowledge/CTO Academy/Data/Semantic Layer.md", "10 Knowledge/CTO Academy/Data/Snowflake Schema.md", "10 Knowledge/CTO Academy/Data/Star Schema.md", diff --git a/10 Knowledge/CTO Academy/Data/Data Lineage.md b/10 Knowledge/CTO Academy/Data/Data Lineage.md new file mode 100644 index 0000000..f75ba77 --- /dev/null +++ b/10 Knowledge/CTO Academy/Data/Data Lineage.md @@ -0,0 +1,673 @@ +**Estimated reading time:** 10–12 minutes + +## Definition + +**Data Lineage** is the ability to understand where data comes from, how it moves, how it changes, and where it is ultimately used. + +It answers questions like: + +> Where did this number come from? + +> Which source system produced it? + +> What transformations were applied? + +> Which reports depend on it? + +A simple lineage path might look like: + +```text +JW Player + ↓ +Raw Viewing Events + ↓ +Bronze + ↓ +Silver + ↓ +Gold + ↓ +[[Fact Table]] + ↓ +[[Semantic Layer]] + ↓ +Power BI Report +``` + +Data Lineage documents that path. + +--- + +## Simple Mental Model + +Think: + +> **Data Lineage = The family tree and travel history of data.** + +It tells you: + +```text +Where it started +↓ +What happened to it +↓ +Where it ended up +``` + +For a KPI, lineage lets you trace backward: + +```text +Executive Dashboard + ↓ +Total Watch Hours + ↓ +Semantic Layer + ↓ +fact_viewing.watch_seconds + ↓ +Gold + ↓ +Silver + ↓ +Bronze + ↓ +JW Player +``` + +--- + +## How It Works + +Imagine a Power BI dashboard displays: + +> Total Watch Hours: 2.4 million + +Someone questions the number. + +Without Data Lineage, the investigation might look like: + +```text +Ask Power BI analyst +↓ +Check report logic +↓ +Ask data engineer +↓ +Check SQL +↓ +Ask platform team +↓ +Check source data +``` + +Nobody has a clear view of the full chain. + +With Data Lineage, you can trace: + +```text +Power BI +↓ +Metric: Total Watch Hours +↓ +[[Semantic Layer]] +↓ +SUM(watch_seconds) / 3600 +↓ +fact_viewing +↓ +Gold transformation +↓ +Silver viewing events +↓ +Bronze raw events +↓ +JW Player API +``` + +Now the investigation is much more structured. + +--- + +## Example + +Suppose your company changes video providers. + +You move from: + +```text +JW Player +``` + +to: + +```text +New Video Platform +``` + +You want to know: + +> Which dashboards will be affected? + +Data Lineage should help answer: + +```text +JW Player + ↓ +Viewing Pipeline + ↓ +fact_viewing + ↓ +Semantic Layer + ↓ +Total Watch Hours + ↓ +Executive Dashboard + ↓ +Content Performance Dashboard + ↓ +Marketing Dashboard +``` + +This is sometimes called **impact analysis**. + +Instead of discovering broken reports after migration, you can identify dependencies beforehand. + +--- + +## How It Fits Into the Bigger Picture + +Now the architecture you've been learning connects like this: + +```text +Source Systems + ↓ +[[ETL vs ELT]] + ↓ +[[Medallion Architecture]] + ↓ +Bronze + ↓ +Silver + ↓ +Gold + ↓ +[[Fact Table]] ++ +[[Dimension Table]] + ↓ +[[Star Schema]] +or +[[Snowflake Schema]] + ↓ +[[Semantic Layer]] + ↓ +Power BI +``` + +[[Data Lineage]] cuts **across the entire architecture**. + +It does not represent one layer. + +It describes the connections between all the layers. + +Conceptually: + +```text +SOURCE → TRANSFORMATION → MODEL → METRIC → REPORT +``` + +That entire chain is lineage. + +--- + +## Technical Lineage vs Business Lineage + +Data Lineage can exist at different levels. + +### Technical Lineage + +Focuses on systems, tables, columns, and transformations. + +Example: + +```text +jw_events.watch_time_ms +↓ +silver_viewing.watch_seconds +↓ +fact_viewing.watch_seconds +↓ +semantic_model.total_watch_hours +``` + +This is useful for: + +- Data engineers + +- Developers + +- Analysts + +- Troubleshooting + + +### Business Lineage + +Focuses on business meaning. + +Example: + +```text +JW Player Viewing Events +↓ +Validated Viewing Data +↓ +Viewing Fact +↓ +Total Watch Hours +↓ +Executive Audience Dashboard +``` + +This is easier for: + +- Managers + +- Executives + +- Data owners + +- Business stakeholders + + +A mature data environment often needs both. + +--- + +## Column-Level Lineage + +Lineage can become very detailed. + +Imagine the final metric: + +```text +Total Watch Hours +``` + +comes from: + +```text +watch_time_ms +``` + +in the source system. + +The transformation could be: + +```text +JW Player +watch_time_ms + ↓ +divide by 1000 + ↓ +watch_seconds + ↓ +divide by 3600 + ↓ +Total Watch Hours +``` + +Column-level lineage tracks that exact transformation. + +This is valuable when someone asks: + +> Why is the number different from the source platform? + +You can see whether: + +- Units were converted + +- Filters were applied + +- Records were excluded + +- Duplicates were removed + +- Business rules were introduced + + +--- + +## Data Lineage and Data Quality + +Imagine a dashboard suddenly shows: + +```text +Total Watch Hours +↓ 40% +``` + +The first question might be: + +> Did audience actually drop? + +But the problem could be: + +```text +JW Player +↓ +API changed +↓ +Pipeline failed +↓ +Silver missing records +↓ +Gold incomplete +↓ +Dashboard shows lower number +``` + +Without lineage, a technical issue can look like a business event. + +With lineage, the team can trace the problem upstream. + +This connects directly to [[Data Quality]]. + +A number can be mathematically correct based on the data it received, while the underlying data is incomplete. + +--- + +## Data Lineage and Data Governance + +[[Data Governance]] asks questions such as: + +> Who owns this data? + +> What does this KPI mean? + +> Who can access it? + +> Can we trust it? + +Data Lineage helps answer: + +> Where did it come from? + +Together: + +```text +[[Data Governance]] + + +[[Data Lineage]] + + +[[Semantic Layer]] + = +More trustworthy analytics +``` + +Governance without lineage can define what a metric should mean without proving how it was produced. + +Lineage without governance can show the technical path without explaining who owns the definition. + +You usually need both. + +--- + +## My Company / Real-World Context + +Imagine the CEO asks: + +> Why does the Power BI dashboard show 1.8 million watch hours while JW Player shows 2.1 million? + +A weak process might involve several days of investigation. + +You ask: + +- Power BI analyst + +- Data engineer + +- JW Player team + +- Vendor + +- Data Governance + + +Each person knows one piece. + +With proper lineage, you could trace: + +```text +JW Player Raw Metric +2.1M hours + ↓ +Bronze + ↓ +Remove invalid events + ↓ +Silver + ↓ +Exclude internal traffic + ↓ +Gold + ↓ +Apply governed viewing definition + ↓ +Semantic Layer + ↓ +Power BI +1.8M hours +``` + +Now the difference may actually be intentional. + +The source platform might count all activity. + +Your governed KPI might exclude: + +- Internal testing + +- Bots + +- Invalid sessions + +- Duplicate events + + +The numbers are different because they answer different questions. + +Lineage helps explain why. + +--- + +## Another Real-World Example: Vendor Migration + +Imagine your company wants to replace JW Player. + +Before signing the new vendor, you should understand what depends on the current one. + +Lineage could reveal: + +```text +JW Player +├── Viewing Analytics +│ ├── Executive Dashboard +│ ├── Content Dashboard +│ └── Audience Report +│ +├── Advertising Metrics +│ └── Revenue Dashboard +│ +└── Content Metadata + ├── Website + └── Mobile App +``` + +Now the migration is no longer: + +> Replace JW Player. + +It is: + +> Replace JW Player while preserving all downstream dependencies. + +That is a much more accurate project scope. + +--- + +## CTO Perspective + +Data Lineage is important for a CTO because modern organizations have interconnected systems. + +A small source change can create unexpected downstream effects. + +The CTO-level questions are: + +> **"If this source changes, what breaks?"** + +> **"If this KPI is wrong, can we trace it back to the source?"** + +> **"Do we know which reports depend on this dataset?"** + +> **"Can we explain how an executive number was produced?"** + +Lineage is also important for: + +- Auditing + +- Compliance + +- Security + +- Data migrations + +- Vendor changes + +- Troubleshooting + +- Impact analysis + + +A mature organization should not depend entirely on one employee's memory to understand how critical data moves. + +The architecture itself should be documented or discoverable. + +--- + +### Questions to Ask + +- What is the original source of this data? + +- Which pipelines transform it? + +- Where does it move through Bronze, Silver, and Gold? + +- Which [[Fact Table|Fact Tables]] and [[Dimension Table|Dimension Tables]] depend on it? + +- Which metrics in the [[Semantic Layer]] use it? + +- Which reports consume those metrics? + +- What happens downstream if this source changes? + +- Can we trace a KPI back to the source field? + +- Are transformations documented? + +- Who owns each stage of the data flow? + +- Is lineage captured automatically or manually? + +- How current is our lineage documentation? + + +--- + +## Meeting Scenario + +**Situation:** + +A vendor announces that an API field your company uses will be removed in 60 days. + +The discussion immediately becomes: + +> "The data team needs to update the API integration." + +**Possible response:** + +> "Before we treat this only as an integration change, can we trace the lineage of that field? I want to know which transformations, metrics, and reports depend on it so we understand the full impact." + +The team discovers: + +```text +API Field +↓ +Silver Transformation +↓ +fact_viewing +↓ +Completion Rate +↓ +Three Power BI Dashboards +↓ +Monthly Executive Report +``` + +You could then respond: + +> "Good. Let's treat this as a dependency change across the full lineage, not just an API ticket. We need to update the source mapping, validate the transformation, confirm the KPI calculation, and test the affected reports before the vendor removes the field." + +You have turned a narrow technical task into a properly scoped change-management problem. + +--- + +## Key Takeaways + +- **Data Lineage** shows where data originates, how it changes, and where it is consumed. + +- Lineage crosses the entire data architecture rather than belonging to one layer. + +- Technical lineage focuses on tables, columns, and transformations. + +- Business lineage focuses on business concepts, metrics, and reports. + +- Column-level lineage can explain exactly how a KPI was calculated. + +- Lineage supports troubleshooting and [[Data Quality]] investigations. + +- Lineage enables impact analysis before source systems or vendors change. + +- [[Data Governance]] defines ownership and meaning; Data Lineage explains the path. + +- A CTO should be able to ask: **"If this changes, what downstream systems and decisions are affected?"** + + +## Related Concepts + +- [[Data Governance]] + +- [[Data Quality]] + +- [[Semantic Layer]] + +- [[Medallion Architecture]] + +- [[ETL vs ELT]] + +- [[Fact Table]] + +- [[Dimension Table]] + +- [[Star Schema]] + +- [[Data Pipeline]] + +- [[Metadata]] + +- [[Data Catalog]] + +- [[Impact Analysis]] + + +The natural next lesson is **Lesson 10: `Data Governance.md`**. Given that this is already part of your actual role, it will be interesting because we'll compare what **Data Governance formally means** against what you're currently doing in practice, including ownership, stewardship, KPI definitions, quality, lineage, and accountability. \ No newline at end of file diff --git a/10 Knowledge/CTO Academy/Data/Quizzes/.gitkeep b/10 Knowledge/CTO Academy/Data/Quizzes/.gitkeep new file mode 100644 index 0000000..e69de29 From 7fbb9ec1f4bfb4296152f84b447a37397543e7db Mon Sep 17 00:00:00 2001 From: modji Date: Tue, 21 Jul 2026 09:15:17 +0200 Subject: [PATCH 2/6] vault backup: 2026-07-21 09:15:17 --- .obsidian/workspace.json | 12 +- .../Quizzes/Quiz 01 - Data Foundations.md | 878 ++++++++++++++++++ 2 files changed, 884 insertions(+), 6 deletions(-) create mode 100644 10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 07b524d..b8605da 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "10 Knowledge/CTO Academy/Data/Data Lineage.md", + "file": "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Data Lineage" + "title": "Quiz 01 - Data Foundations" } } ] @@ -196,10 +196,11 @@ "omnisearch:Omnisearch": false } }, - "active": "92c89fdf4eb1874e", + "active": "579a503273b0c5ec", "lastOpenFiles": [ - "10 Knowledge/CTO Academy/Data/Quizzes", "10 Knowledge/CTO Academy/Data/Data Lineage.md", + "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", + "10 Knowledge/CTO Academy/Data/Quizzes", "10 Knowledge/CTO Academy/Data/Semantic Layer.md", "10 Knowledge/CTO Academy/Data/Snowflake Schema.md", "10 Knowledge/CTO Academy/Data/Star Schema.md", @@ -230,7 +231,6 @@ "90 Archives", "40 Journal", "30 Personal Projects/Modji", - "30 Personal Projects", - "20 Work/Ideas" + "30 Personal Projects" ] } \ No newline at end of file diff --git a/10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md b/10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md new file mode 100644 index 0000000..cc84c55 --- /dev/null +++ b/10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md @@ -0,0 +1,878 @@ + + +**Score: 20.5 / 24 — 85%** + +## Overall Assessment + +### Strongest Areas + +- [[Medallion Architecture]] + +- [[ETL vs ELT]] + +- [[Grain]] + +- [[Fact Table]] + +- [[Semantic Layer]] + +- CTO-level decision-making and architectural judgment + + +### Areas to Reinforce + +- [[Data Lake]] vs [[Data Warehouse]] + +- [[Fact Table]] vs [[Dimension Table]] + +- [[Star Schema]] vs [[Snowflake Schema]] + +- [[Data Lineage]] + + +Most of my mistakes were about precise definitions and architectural trade-offs rather than the overall architecture. + +--- + +# Section A - Multiple Choice + +## Question 1 + +**Question:** What is the best description of the Gold layer? + +A. The place where all raw source data is stored unchanged +B. The layer where data is cleaned technically but has no business meaning +C. The layer containing business-ready data intended for consumption +D. The reporting tool used to visualize the data + +**My Answer:** C + +**Result:** ✅ Correct + +--- + +## Question 2 + +**Question:** Which statement is correct? + +A. A Data Lake is always Bronze and a Data Warehouse is always Gold +B. Bronze, Silver, and Gold describe data maturity, while Data Lake and Data Warehouse describe architecture/storage approaches +C. Azure Synapse is another name for the Gold layer +D. A Lakehouse is simply a Data Warehouse hosted in Azure + +**My Answer:** B + +**Result:** ✅ Correct + +--- + +## Question 3 + +**Question:** What is the main difference between ETL and ELT? + +A. ETL is used only in the cloud +B. ELT does not transform data +C. ETL transforms before loading, while ELT loads before transforming +D. ETL uses Fact Tables while ELT uses Dimension Tables + +**My Answer:** C + +**Result:** ✅ Correct + +--- + +## Question 4 + +**Question:** A table contains one row for every individual video viewing session. What does this describe? + +A. The Semantic Layer +B. The Grain +C. The Data Lineage +D. The Dimension + +**My Answer:** B + +**Result:** ✅ Correct + +--- + +## Question 5 + +**Question:** Which is most likely a Fact Table? + +A. `dim_platform` +B. `dim_date` +C. `fact_viewing` +D. `dim_content` + +**My Answer:** C + +**Result:** ✅ Correct + +--- + +## Question 6 + +**Question:** Which is most likely a Dimension Table? + +A. A table containing one row per video viewing event +B. A table containing content title, genre, series, and language +C. A table containing total watch seconds per session +D. A table containing raw API events + +**My Answer:** B + +**Result:** ✅ Correct + +--- + +## Question 7 + +**Question:** In a Star Schema: + +A. Every dimension connects to another dimension before reaching the Fact Table +B. The Fact Table is normally at the center and dimensions connect directly to it +C. There are no Fact Tables +D. All data is stored in one large table + +**My Answer:** B + +**Result:** ✅ Correct + +--- + +## Question 8 + +**Question:** Compared with a Star Schema, a Snowflake Schema generally: + +A. Has fewer tables and fewer joins +B. Stores only raw data +C. Further normalizes Dimension Tables into related tables +D. Eliminates the need for a Fact Table + +**My Answer:** C + +**Result:** ✅ Correct + +--- + +## Question 9 + +**Question:** What is the main purpose of a Semantic Layer? + +A. Store raw data +B. Replace the Data Warehouse +C. Provide consistent business definitions and reusable metrics +D. Manage SSL certificates + +**My Answer:** C + +**Result:** ✅ Correct + +--- + +## Question 10 + +**Question:** What does Data Lineage primarily help answer? + +A. How attractive a dashboard looks +B. Where data came from, how it changed, and where it is used +C. Which database technology is cheapest +D. How many Dimension Tables should exist + +**My Answer:** B + +**Result:** ✅ Correct + +--- + +# Section B - Short Response + +## Question 11 + +**Question:** Explain the difference between Bronze and Silver. + +### My Answer + +> Bronze is the raw data. Silver is a cleaned up data but not yet fully ready for business use. + +### Result + +✅ **Correct** + +### Key Point + +[[Bronze Layer]] contains raw or minimally processed source data. + +[[Silver Layer]] contains cleaned, validated, standardized, or enriched data that is more trustworthy but not necessarily structured for final business consumption. + +--- + +## Question 12 + +**Question:** Explain the difference between a Data Lake and a Data Warehouse. + +### My Answer + +> Data lake collects all raw data from every source. While the Data Warehouse has the data more organized. Hesitant to even say that data is transformed for the warehouse in ETL. + +### Result + +🟡 **Partially Correct** + +### Correction + +A [[Data Lake]] can store large amounts of **raw and processed data** in many formats. It is not limited to raw data. + +A [[Data Warehouse]] contains highly structured data designed primarily for analytics and reporting. + +The important distinction is: + +> **Data Lake = flexible storage for diverse data.** + +> **Data Warehouse = structured analytical system optimized for querying and reporting.** + +Data may be transformed before entering a Data Warehouse in an ETL architecture, but with modern ELT architectures, data can also be loaded first and transformed afterward. + +### What I Need to Remember + +Do not automatically think: + +> Data Lake = Raw +> Data Warehouse = Transformed + +Instead: + +> **Lake vs Warehouse = architecture/storage approach** + +> **Bronze vs Silver vs Gold = data maturity** + +--- + +## Question 13 + +**Question:** What does this question mean? + +> "What is the grain of this Fact Table?" + +### My Answer + +> It is a question asking what a single row in this Fact Table means. + +### Result + +✅ **Correct** + +### Key Point + +[[Grain]] defines exactly what **one row represents** in a [[Fact Table]]. + +Example: + +> One row = one viewing session for one piece of content by one user. + +--- + +## Question 14 + +**Question:** Explain the relationship between a Fact Table and a Dimension Table. + +### My Answer + +> Dimension are data that cannot be aggregated and provide context to those tables that can be aggregated. The later are facts table. + +### Result + +🟡 **Partially Correct** + +### Correction + +A [[Fact Table]] records **business events or measurable observations** and commonly contains [[Measure|Measures]] such as revenue, watch seconds, quantity, or impressions. + +A [[Dimension Table]] provides **descriptive context** about those events. + +For example: + +`fact_viewing` tells us: + +> A viewing event happened for 125 seconds. + +Dimensions tell us: + +> **Who** watched? +> **What** was watched? +> **When**? +> **Where or how**? + +The distinction is not simply that Facts can be aggregated and Dimensions cannot. Dimension Tables can contain numeric attributes too. + +The important distinction is their **role in the analytical model**. + +### What I Need to Remember + +> **Fact = What happened?** + +> **Dimension = Describe what happened.** + +--- + +## Question 15 + +**Question:** Why might a business choose a Star Schema instead of a Snowflake Schema for Power BI? + +### My Answer + +> Not sure how to answer this, but snowflake would be used to respond specific business questions. Often snowflake will provide deeper analysis since the database is normalized. + +### Result + +❌ **Incorrect** + +### Correction + +A business might choose a [[Star Schema]] instead of a [[Snowflake Schema]] for Power BI because a Star Schema generally has: + +- Fewer tables + +- Fewer joins + +- Simpler relationships + +- Easier navigation for analysts + +- A model that is easier for BI tools and users to understand + + +A Snowflake Schema is more normalized, but this does **not** mean it provides deeper analysis. + +Both models can potentially answer the same business questions. + +The primary trade-off is: + +> **Star Schema = simplicity** + +> **Snowflake Schema = normalization and reduced duplication** + +### What I Need to Remember + +Normalization does not automatically mean better or deeper analytics. + +For BI: + +> **Simple models are often preferable unless additional complexity solves a real problem.** + +--- + +## Question 16 + +**Question:** Why can two Power BI dashboards show different values for the same KPI even if neither report contains a technical error? + +### My Answer + +> The semantic layer must be reviewed to ensure that the metrics in all reports have the same definitions. + +### Result + +✅ **Correct** + +### Key Point + +Two reports can be technically correct while using different business definitions. + +A governed [[Semantic Layer]] helps ensure that a KPI is defined once and reused consistently across reports. + +--- + +## Question 17 + +**Question:** Give one example of how Data Lineage could help during a vendor migration. + +### My Answer + +> Not sure.. all I can think of is the impact on a project that this could have since it defines the scope better. + +### Result + +🟡 **Partially Correct** + +### Correction + +This answer was heading in the right direction. + +During a vendor migration, [[Data Lineage]] helps identify **everything downstream that depends on the vendor's data**. + +For example: + +```text +JW Player +↓ +Viewing Pipeline +↓ +fact_viewing +↓ +Total Watch Hours +↓ +Semantic Layer +↓ +Executive Dashboard +↓ +Content Performance Dashboard +``` + +Before replacing JW Player, Data Lineage helps determine which: + +- Pipelines + +- Transformations + +- Tables + +- Metrics + +- Reports + + +will be affected. + +This does help define the true project scope through [[Impact Analysis]]. + +### What I Need to Remember + +The key CTO question is: + +> **"If we change this system, what downstream systems, data, metrics, and reports will be affected?"** + +--- + +# Section C - Scenario Questions + +## Question 18 - Audience vs Sessions + +**Question:** + +An analyst presents: + +> "Our audience increased by 30%." + +You ask how audience was calculated. + +They answer: + +> "We counted the number of rows in `fact_viewing`. One row represents one viewing session." + +What is wrong with the analyst's statement, and what should the metric probably be called? + +### My Answer + +> A viewing doesn't represent an audience. We must first agree to what is an audience. A multiple viewing could be linked to one person. + +### Result + +✅ **Correct** + +### Key Point + +If the [[Grain]] is one row per viewing session, counting rows measures **viewing sessions**, not necessarily audience. + +One person can generate multiple viewing sessions. + +The metric should likely be called: + +> **Viewing Sessions** + +A separate governed definition is needed for: + +> **Audience** or **Unique Viewers** + +--- + +## Question 19 - Competing KPI Definitions + +**Question:** + +Marketing defines: + +> Active User = anyone who opened the website + +Product defines: + +> Active User = anyone who watched at least one video + +Both teams publish dashboards showing "Active Users." + +What architectural or governance problem do you see, and what concept from the lessons could help solve it? + +### My Answer + +> A semantic layer where the definition of items is common to all. Basically, create a governed shared business definition. + +### Result + +✅ **Correct** + +### Key Point + +The organization has competing definitions of the same business metric. + +A governed [[Semantic Layer]], supported by [[Data Governance]], should establish a shared definition of **Active User** that can be consistently reused. + +--- + +## Question 20 - API Change + +**Question:** + +A vendor announces that the field `watch_time_ms` will be removed from its API. + +Before approving the technical change, what should you ask the team to determine? + +Give 2 or 3 things. + +### My Answer + +> Don't know the answer.. has to do with data lineage but I didn't read it properly. + +### Result + +❌ **Incorrect** + +### Correction + +This is a [[Data Lineage]] and [[Impact Analysis]] question. + +I should ask the team to determine: + +1. **Where is this field used downstream?** + + +Which pipelines, Bronze/Silver/Gold transformations, [[Fact Table|Fact Tables]], or other datasets depend on it? + +2. **Which metrics depend on it?** + + +For example: + +```text +watch_time_ms +↓ +watch_seconds +↓ +Total Watch Hours +``` + +3. **Which reports or business processes will be affected?** + + +For example: + +```text +Total Watch Hours +↓ +Executive Dashboard +↓ +Content Performance Report +``` + +The goal is to understand the complete impact **before** treating it as simply an API change. + +### What I Need to Remember + +When a source changes: + +> **Trace forward.** + +```text +Source +↓ +Pipeline +↓ +Data Model +↓ +Metric +↓ +Report +``` + +When a KPI looks wrong: + +> **Trace backward.** + +```text +Report +↑ +Metric +↑ +Data Model +↑ +Pipeline +↑ +Source +``` + +This is [[Data Lineage]]. + +--- + +# Section D - CTO Judgment + +## Question 21 + +**Question:** A data architect says: + +> "We should move to a Lakehouse because it's the modern architecture." + +What is the best response? + +A. "Yes, let's start immediately." +B. "No, Data Warehouses are better." +C. "What problem are we trying to solve that our current architecture cannot solve?" +D. "Let's ask Power BI which architecture it prefers." + +**My Answer:** C + +**Result:** ✅ Correct + +### Key Point + +Architecture should solve a problem. + +Do not adopt technology simply because it is newer. + +--- + +## Question 22 + +**Question:** An architect wants to split one Dimension Table into six smaller tables because: + +> "It's more normalized and technically cleaner." + +What is the strongest CTO-style response? + +A. "Normalization is always better, so let's do it." +B. "How does the additional complexity improve governance, maintainability, performance, or business outcomes?" +C. "Snowflake Schemas are bad." +D. "Let the developers decide without discussing it." + +**My Answer:** B + +**Result:** ✅ Correct + +### Key Point + +Architectural complexity should provide measurable value. + +Technical elegance alone is not sufficient justification. + +--- + +## Question 23 + +**Question:** A dashboard shows a sudden 40% drop in watch hours. + +What should you investigate before concluding that audience behavior changed? + +A. Whether the dashboard colors changed +B. Whether the source, pipeline, transformations, or upstream data quality changed +C. Whether the CEO likes the metric +D. Whether a new Dimension Table should be created + +**My Answer:** B + +**Result:** ✅ Correct + +### Key Point + +A technically correct dashboard can display incorrect or incomplete business information if something upstream has failed. + +Use [[Data Lineage]] to trace backward and investigate [[Data Quality]]. + +--- + +## Question 24 + +**Question:** Two reports disagree on platform performance because each analyst created their own mapping of Roku, Apple TV, and Android TV. + +What is the strongest long-term solution? + +A. Let each analyst keep their own definition +B. Pick whichever dashboard has the higher number +C. Create a governed shared platform dimension or shared business definition +D. Merge all the reports into one giant spreadsheet + +**My Answer:** C + +**Result:** ✅ Correct + +### Key Point + +Shared business concepts should be governed centrally rather than recreated independently in individual reports. + +This connects: + +- [[Dimension Table]] + +- [[Semantic Layer]] + +- [[Data Governance]] + + +--- + +# Bonus - Question 25 + +**Question:** + +In one sentence, explain the entire path from source system to executive dashboard using as many of these concepts as you can: + +`ETL/ELT`, `Bronze`, `Silver`, `Gold`, `Fact Table`, `Dimension Table`, `Star Schema`, `Semantic Layer`, `Power BI`, `Data Lineage` + +### My Answer + +> Not answered. + +### Result + +➖ **No penalty** + +--- + +# Final Result + +**Score: 20.5 / 24** + +**Percentage: 85%** + +## What I Understand Well + +I have a strong understanding of the overall data architecture: + +```text +Source Systems +↓ +[[ETL vs ELT]] +↓ +[[Medallion Architecture]] +↓ +Bronze → Silver → Gold +↓ +Analytical Data Model +↓ +[[Fact Table]] + [[Dimension Table]] +↓ +[[Star Schema]] / [[Snowflake Schema]] +↓ +[[Semantic Layer]] +↓ +Power BI +↓ +Business Decisions +``` + +I also understand the CTO-level principle of asking: + +> **"What problem are we trying to solve?"** + +before choosing an architecture or technology. + +## What I Need to Reinforce + +### 1. Data Lake vs Data Warehouse + +Remember: + +> **Data Lake vs Data Warehouse = architecture/storage** + +> **Bronze vs Silver vs Gold = data maturity** + +A Data Lake is not automatically Bronze. + +A Data Warehouse is not automatically Gold. + +--- + +### 2. Fact vs Dimension + +Remember: + +> **Fact = What happened?** + +> **Dimension = Describe what happened.** + +Do not define the difference only as: + +> Facts contain numbers; dimensions do not. + +--- + +### 3. Star vs Snowflake + +Remember: + +```text +[[Star Schema]] +Simple +Fewer joins +BI-friendly +Some duplication is acceptable + +vs. + +[[Snowflake Schema]] +More normalized +More tables +More joins +Less duplication +Potentially more complexity +``` + +Normalization does **not** mean deeper analysis. + +Both can answer complex business questions. + +--- + +### 4. Data Lineage + +Remember two directions: + +**Something is wrong with a KPI?** + +```text +Dashboard +↑ +Metric +↑ +Model +↑ +Gold +↑ +Silver +↑ +Bronze +↑ +Source + +TRACE BACKWARD +``` + +**A source or vendor is changing?** + +```text +Source +↓ +Pipeline +↓ +Model +↓ +Metric +↓ +Reports + +TRACE FORWARD +``` + +The CTO-level question is: + +> **"If this changes, what downstream systems and decisions are affected?"** + +That is [[Impact Analysis]]. \ No newline at end of file From 2d0e690e713296e5abd9e77eabbbc74cfa62e25b Mon Sep 17 00:00:00 2001 From: modji Date: Tue, 21 Jul 2026 20:40:18 +0200 Subject: [PATCH 3/6] vault backup: 2026-07-21 20:40:18 --- .obsidian/workspace.json | 5 +- .../CTO Academy/Data/Data Governance.md | 774 ++++++++++++++++++ 2 files changed, 777 insertions(+), 2 deletions(-) create mode 100644 10 Knowledge/CTO Academy/Data/Data Governance.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index b8605da..4b05440 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", + "file": "10 Knowledge/CTO Academy/Data/Data Governance.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Quiz 01 - Data Foundations" + "title": "Data Governance" } } ] @@ -199,6 +199,7 @@ "active": "579a503273b0c5ec", "lastOpenFiles": [ "10 Knowledge/CTO Academy/Data/Data Lineage.md", + "10 Knowledge/CTO Academy/Data/Data Governance.md", "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", "10 Knowledge/CTO Academy/Data/Quizzes", "10 Knowledge/CTO Academy/Data/Semantic Layer.md", diff --git a/10 Knowledge/CTO Academy/Data/Data Governance.md b/10 Knowledge/CTO Academy/Data/Data Governance.md new file mode 100644 index 0000000..95ff566 --- /dev/null +++ b/10 Knowledge/CTO Academy/Data/Data Governance.md @@ -0,0 +1,774 @@ +**Estimated reading time:** 12–15 minutes + +# Data Governance + +## Definition + +**Data Governance** is the framework of **people, processes, policies, and standards** that ensures data is: + +- Accurate + +- Consistent + +- Secure + +- Trusted + +- Well-defined + +- Properly managed throughout its lifecycle + + +A common misconception is: + +> "Data Governance is about data." + +It isn't. + +> **Data Governance is about how an organization manages its data.** + +Think of it this way: + +The data itself doesn't become trustworthy. + +The **organization makes it trustworthy**. + +--- + +## Simple Mental Model + +Think of Data Governance as the rules of a city. + +The city already exists. + +People already drive. + +Buildings already exist. + +Governance answers questions like: + +- Who owns this road? + +- Who can build here? + +- Who maintains the traffic lights? + +- What happens if something breaks? + +- Who decides the speed limit? + + +Without those rules, the city still functions… + +...just not very well. + +Data is the same. + +--- + +## How It Works + +Data Governance is built around several fundamental pillars. + +```text + DATA GOVERNANCE + + ┌────────────────────┐ + │ Business Ownership │ + └────────────────────┘ + │ + ▼ + Shared Definitions + │ + ▼ + Data Quality + │ + ▼ + Data Security + │ + ▼ + Data Lineage + │ + ▼ + Metadata + │ + ▼ + Stewardship +``` + +These pillars work together. + +Weakness in one usually affects the others. + +--- + +# Pillar 1 — Ownership + +One of the first governance questions is: + +> **Who owns this data?** + +Not: + +> Who built the report? + +Not: + +> Who manages SQL? + +Ownership is a **business responsibility**, not a technical one. + +Example: + +```text +Employee Data +``` + +Owner: + +Human Resources + +Not IT. + +Example: + +```text +Financial Data +``` + +Owner: + +Finance. + +Not the DBA. + +IT manages systems. + +The business owns the meaning. + +--- + +## Example + +Imagine your company has: + +```text +Unique Viewer +``` + +Marketing says: + +> "It's anyone who visits." + +Product says: + +> "It's anyone who watches." + +Engineering says: + +> "It's a unique device." + +Who's correct? + +Governance says: + +> **There must be one owner responsible for defining the KPI.** + +Without ownership: + +Everyone has an opinion. + +Nobody has responsibility. + +--- + +# Pillar 2 — Business Definitions + +Governance also answers: + +> What exactly does this KPI mean? + +For example: + +Revenue. + +Simple? + +Not really. + +Does Revenue mean: + +- Gross Revenue + +- Net Revenue + +- Before refunds + +- After refunds + +- Before tax + +- After tax + + +Same word. + +Potentially six different metrics. + +This is why we studied [[Semantic Layer]]. + +A Semantic Layer implements business definitions. + +Governance decides those definitions. + +--- + +# Pillar 3 — Data Quality + +Governance also asks: + +Can we trust the data? + +Quality includes things like: + +Accuracy + +Completeness + +Consistency + +Timeliness + +Validity + +Uniqueness + +Example: + +Suppose: + +```text +Platform + +Roku +roku +ROKU +Connected TV +``` + +Technically valid? + +Yes. + +Business useful? + +Not really. + +Governance establishes standards. + +--- + +# Pillar 4 — Security + +Governance determines: + +Who may access which data? + +Examples: + +HR salaries + +Medical information + +Financial forecasts + +Personally identifiable information + +Not everyone should access everything. + +Governance establishes policies. + +Technology enforces them. + +--- + +# Pillar 5 — Data Lineage + +As we learned in [[Data Lineage]], governance also requires understanding: + +Where data came from + +How it changed + +Where it's used + +Example: + +```text +Executive KPI + +↓ + +Semantic Layer + +↓ + +Fact Table + +↓ + +Gold + +↓ + +Silver + +↓ + +Bronze + +↓ + +Source System +``` + +If you cannot explain where a KPI came from… + +Governance is incomplete. + +--- + +# Pillar 6 — Metadata + +Metadata simply means: + +> Data about data. + +Examples: + +Table owner + +Description + +Refresh schedule + +Business definition + +Data sensitivity + +Last refresh + +Source system + +Without metadata: + +People ask: + +> "What does this table do?" + +With metadata: + +The answer already exists. + +--- + +# Pillar 7 — Stewardship + +Ownership and stewardship are different. + +Owner + +Responsible for: + +Business meaning. + +Steward + +Responsible for: + +Maintaining quality. + +Example: + +HR owns employee data. + +A Data Steward may ensure: + +Employee IDs are valid. + +Departments exist. + +Names follow standards. + +Quality checks succeed. + +--- + +## How It Fits Into the Bigger Picture + +Everything you've learned now connects. + +```text +Source Systems + +↓ + +[[ETL vs ELT]] + +↓ + +[[Medallion Architecture]] + +↓ + +Bronze + +↓ + +Silver + +↓ + +Gold + +↓ + +[[Fact Table]] ++ +[[Dimension Table]] + +↓ + +[[Star Schema]] + +↓ + +[[Semantic Layer]] + +↓ + +Power BI + +↓ + +Business Decisions + +══════════════════════════════ + +[[Data Governance]] + +applies across ALL layers. +``` + +Governance isn't another technical layer. + +It's a management framework that influences every layer. + +--- + +## Governance vs Management + +People often confuse: + +Project Management + +Data Management + +Data Governance + +Think: + +Project Management + +Builds things. + +Data Management + +Operates systems. + +Data Governance + +Defines the rules. + +Different responsibilities. + +--- + +## Governance vs Data Quality + +These are not the same. + +Governance asks: + +Should this data exist? + +Who owns it? + +Who defines it? + +How should quality be measured? + +Data Quality asks: + +Is today's data actually good? + +Governance creates expectations. + +Quality measures whether those expectations were met. + +--- + +## Governance vs Security + +Governance says: + +Finance salaries should only be visible to Finance leadership. + +Security implements: + +Active Directory + +Azure AD + +Permissions + +Row-Level Security + +Encryption + +Governance defines policy. + +Technology enforces it. + +--- + +## My Company / Real-World Context + +This lesson is probably the closest to your day-to-day job. + +Think about your previous conversations with MH regarding demographics. + +Marketing wanted: + +Age + +Gender + +Unknown % + +The disagreement wasn't technical. + +It was governance. + +Questions included: + +- What does Unknown actually represent? + +- Is the metric acceptable? + +- Who owns the definition? + +- Can it be improved? + +- How should executives interpret it? + + +Another example: + +Imagine your two analysts create: + +```text +Completion Rate +``` + +One uses: + +95%. + +Another uses: + +90%. + +Neither calculation is technically wrong. + +Governance says: + +Choose one. + +Document it. + +Implement it once. + +Reuse it everywhere. + +That is exactly what your team should eventually own. + +--- + +## CTO Perspective + +Many new CTOs spend their time discussing: + +Cloud + +AI + +Microservices + +Containers + +Kubernetes + +Those are important. + +But organizations rarely fail because they chose the wrong container technology. + +They often fail because: + +Nobody trusts the numbers. + +Different reports disagree. + +Nobody owns the KPI. + +Nobody knows where data came from. + +Nobody knows what changes will break. + +The CTO's role is to create an environment where executives trust the data enough to make decisions. + +That trust comes from governance. + +--- + +### Questions to Ask + +When someone presents a KPI: + +- Who owns this metric? + +- What is its business definition? + +- Is it documented? + +- Where is it calculated? + +- Does everyone use the same definition? + +- Can we trace it through [[Data Lineage]]? + +- What is the data quality? + +- Who approves changes? + +- Who has access? + +- What happens if the source changes? + + +--- + +## Meeting Scenario + +**Situation** + +The CEO asks: + +> "Why does Marketing say we have 2.3 million active users while Product says 1.8 million?" + +Everyone immediately starts debating SQL. + +A CTO-style response would be: + +> "Before comparing the SQL, I'd like us to confirm whether we're using the same business definition of Active User. If each department has created its own definition, then the issue isn't technical—it's a governance issue. Let's establish one owner for this KPI, agree on the definition, document it in our Semantic Layer, and ensure every report consumes the same metric." + +Notice what happened. + +You didn't ask: + +> Which report is wrong? + +You asked: + +> Why does the organization allow two different definitions of the same KPI? + +That's governance thinking. + +--- + +## Key Takeaways + +- **Data Governance** is the framework that ensures organizational trust in data. + +- Governance is about **people, policies, ownership, and standards**, not just technology. + +- Data Governance spans the entire data architecture. + +- Every important KPI should have a clearly identified business owner. + +- The [[Semantic Layer]] implements business definitions; Governance decides those definitions. + +- [[Data Lineage]] explains where data comes from; Governance ensures it is understood and managed. + +- Governance defines policy; security technologies enforce it. + +- High-quality analytics require both good technical architecture and strong governance. + + +## Related Concepts + +- [[Semantic Layer]] + +- [[Data Lineage]] + +- [[Data Quality]] + +- [[Metadata]] + +- [[Data Steward]] + +- [[Data Owner]] + +- [[Business Glossary]] + +- [[KPI]] + +- [[Measure]] + +- [[Fact Table]] + +- [[Dimension Table]] + +- [[Medallion Architecture]] + + +--- + +## 🎯 CTO Reflection + +This is actually where your career and these lessons begin to merge. + +You told me several weeks ago that you wanted to become a CTO, and that you didn't necessarily want to be the best programmer anymore—you wanted to become someone who **leads technology to produce business results**. + +This lesson is the first one where your current job gives you a significant advantage. + +When I look at the work you've described over the past months, you've already been doing pieces of Data Governance without calling it that: + +- Challenging inconsistent KPI definitions. + +- Asking departments to agree on business meanings. + +- Building executive dashboards. + +- Thinking about ownership instead of just SQL. + +- Questioning whether a metric is actually measuring what its name implies. + +- Wanting reusable definitions instead of every analyst inventing their own. + + +That's not just reporting. + +That's the beginning of governance. + +From here, the academy shifts slightly. We've built the technical foundation. The next lessons will increasingly connect **architecture, leadership, governance, and decision-making**—the areas that distinguish a CTO from a senior developer or data engineer. I think that's where the material will become even more directly applicable to your day-to-day role. \ No newline at end of file From 5aee490718fe44f9c4cffe6333dbaa81ac8c8e9b Mon Sep 17 00:00:00 2001 From: modji Date: Tue, 21 Jul 2026 21:06:00 +0200 Subject: [PATCH 4/6] vault backup: 2026-07-21 21:06:00 --- .obsidian/workspace.json | 17 +++++++++-------- .../CTO Academy/Data/Data Governance.md | 16 ++++------------ 10 Knowledge/Templates/Untitled.md | 0 20 Work/Weekly Review/.gitkeep | 0 4 files changed, 13 insertions(+), 20 deletions(-) create mode 100644 10 Knowledge/Templates/Untitled.md create mode 100644 20 Work/Weekly Review/.gitkeep diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 4b05440..6b02182 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "10 Knowledge/CTO Academy/Data/Data Governance.md", + "file": "10 Knowledge/Templates/Untitled.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Data Governance" + "title": "Untitled" } } ] @@ -198,8 +198,13 @@ }, "active": "579a503273b0c5ec", "lastOpenFiles": [ - "10 Knowledge/CTO Academy/Data/Data Lineage.md", + "10 Knowledge/Templates/Employe - Team Template.md", + "10 Knowledge/Templates/Untitled.md", + "20 Work/Weekly Review", + "10 Knowledge/Templates/Meeting Template.md", + "10 Knowledge/Templates/Project Template.md", "10 Knowledge/CTO Academy/Data/Data Governance.md", + "10 Knowledge/CTO Academy/Data/Data Lineage.md", "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", "10 Knowledge/CTO Academy/Data/Quizzes", "10 Knowledge/CTO Academy/Data/Semantic Layer.md", @@ -213,10 +218,7 @@ "20 Work/Ideas/Coaching on management.md", "10 Knowledge/Templates/Journal Template.md", "10 Knowledge/Templates/Academy Template.md", - "10 Knowledge/Templates/Employe - Team Template.md", - "10 Knowledge/Templates/Project Template.md", "10 Knowledge/Templates/Idea Template.md", - "10 Knowledge/Templates/Meeting Template.md", "10 Knowledge/Templates/Decision Template.md", "10 Knowledge/Templates", "10 Knowledge/CTO Academy/Data/Data Lake vs Data Warehouse vs Lakehouse.md", @@ -231,7 +233,6 @@ "50 Principles", "90 Archives", "40 Journal", - "30 Personal Projects/Modji", - "30 Personal Projects" + "30 Personal Projects/Modji" ] } \ No newline at end of file diff --git a/10 Knowledge/CTO Academy/Data/Data Governance.md b/10 Knowledge/CTO Academy/Data/Data Governance.md index 95ff566..0c8ba87 100644 --- a/10 Knowledge/CTO Academy/Data/Data Governance.md +++ b/10 Knowledge/CTO Academy/Data/Data Governance.md @@ -1,7 +1,5 @@ **Estimated reading time:** 12–15 minutes -# Data Governance - ## Definition **Data Governance** is the framework of **people, processes, policies, and standards** that ensures data is: @@ -478,17 +476,11 @@ Data Governance Think: -Project Management +Project Management -> Builds things. -Builds things. +Data Management -> Operates systems. -Data Management - -Operates systems. - -Data Governance - -Defines the rules. +Data Governance -> Defines the rules. Different responsibilities. @@ -546,7 +538,7 @@ Technology enforces it. This lesson is probably the closest to your day-to-day job. -Think about your previous conversations with MH regarding demographics. +Think about your previous conversations with Marketing regarding demographics. Marketing wanted: diff --git a/10 Knowledge/Templates/Untitled.md b/10 Knowledge/Templates/Untitled.md new file mode 100644 index 0000000..e69de29 diff --git a/20 Work/Weekly Review/.gitkeep b/20 Work/Weekly Review/.gitkeep new file mode 100644 index 0000000..e69de29 From 070196df8d6177183af14f6b8816f8c8d200e264 Mon Sep 17 00:00:00 2001 From: modji Date: Tue, 21 Jul 2026 22:01:27 +0200 Subject: [PATCH 5/6] vault backup: 2026-07-21 22:01:27 --- .obsidian/plugins/iconic/data.json.backup2 | 24 +- .obsidian/workspace.json | 6 +- 10 Knowledge/Templates/Untitled.md | 0 .../Templates/Weekly Review Template.md | 259 ++++++++++++++++++ 4 files changed, 285 insertions(+), 4 deletions(-) delete mode 100644 10 Knowledge/Templates/Untitled.md create mode 100644 10 Knowledge/Templates/Weekly Review Template.md diff --git a/.obsidian/plugins/iconic/data.json.backup2 b/.obsidian/plugins/iconic/data.json.backup2 index ed37cf0..235dd48 100644 --- a/.obsidian/plugins/iconic/data.json.backup2 +++ b/.obsidian/plugins/iconic/data.json.backup2 @@ -28,7 +28,29 @@ }, "appIcons": {}, "tabIcons": {}, - "fileIcons": {}, + "fileIcons": { + "00 Inbox": { + "icon": "lucide-inbox" + }, + "10 Knowledge": { + "icon": "lucide-school" + }, + "20 Work": { + "icon": "lucide-building" + }, + "30 Personal Projects": { + "icon": "lucide-user" + }, + "40 Journal": { + "icon": "lucide-pen-tool" + }, + "50 Principles": { + "icon": "lucide-star" + }, + "90 Archives": { + "icon": "lucide-archive" + } + }, "bookmarkIcons": {}, "tagIcons": {}, "propertyIcons": {}, diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 6b02182..b2071fa 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "10 Knowledge/Templates/Untitled.md", + "file": "10 Knowledge/Templates/Weekly Review Template.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Untitled" + "title": "Weekly Review Template" } } ] @@ -199,7 +199,7 @@ "active": "579a503273b0c5ec", "lastOpenFiles": [ "10 Knowledge/Templates/Employe - Team Template.md", - "10 Knowledge/Templates/Untitled.md", + "10 Knowledge/Templates/Weekly Review Template.md", "20 Work/Weekly Review", "10 Knowledge/Templates/Meeting Template.md", "10 Knowledge/Templates/Project Template.md", diff --git a/10 Knowledge/Templates/Untitled.md b/10 Knowledge/Templates/Untitled.md deleted file mode 100644 index e69de29..0000000 diff --git a/10 Knowledge/Templates/Weekly Review Template.md b/10 Knowledge/Templates/Weekly Review Template.md new file mode 100644 index 0000000..c7ff456 --- /dev/null +++ b/10 Knowledge/Templates/Weekly Review Template.md @@ -0,0 +1,259 @@ +# Weekly Review - YYYY-MM-DD (Week XX) + +## 🎯 Executive Summary (2-3 sentences) + +Overall health of the department: + +Traffic light: 🟢 / 🟡 / 🔴 + +This week's headline: +- + +--- + +# Wins 🏆 + +Things that moved the company forward. + +- +- +- + +--- + +# Risks ⚠️ + +What keeps me awake? + +| Risk | Impact | Mitigation | Owner | +|------|--------|------------|-------| +| | | | | + +--- + +# Team 👥 + +## Team health + +Overall morale: +🟢 High +🟡 Normal +🔴 Low + +### Coaching / Recognition + +Who deserves recognition? + +- + +Who needs coaching? + +- + +Any conflicts? + +- + +Any hiring concerns? + +- + +--- + +# Projects 📁 + +| Project | Status | Next Milestone | Blocker | +|----------|--------|----------------|----------| +| | 🟢 | | | +| | 🟡 | | | +| | 🔴 | | | + +Projects slipping: + +- + +Projects completed: + +- + +--- + +# Vendors 🤝 + +Any issue? + +Renewals coming? + +Invoices pending? + +Contract risks? + +- + +--- + +# KPIs 📈 + +Important numbers this week. + +- + +- + +- + +Anything unusual? + +- + +--- + +# Decisions Made + +Decision | Why +---------|---- +| | + +--- + +# Decisions Needed + +What decisions must be made next week? + +- + +- + +--- + +# Meetings + +Which meetings created value? + +- + +Which meetings should disappear? + +- + +Which meetings could become: + +- Slack +- Email +- Dashboard +- Monthly instead of weekly + +--- + +# Delegation + +What did I unnecessarily do myself? + +- + +What should be delegated next week? + +- + +--- + +# Learning + +Books + +Articles + +Podcasts + +Lessons learned + +- + +--- + +# Process Improvements + +Something that annoyed me this week. + +How can I make sure it never happens again? + +- + +--- + +# CEO Questions + +If the CEO walked in Monday morning, what questions might they ask? + +1. + +2. + +3. + +Do I already know the answers? + +--- + +# Top Priorities Next Week + +1. + +2. + +3. + +--- + +# Personal Reflection + +Energy (1-10): + +Stress (1-10): + +Biggest accomplishment: + +Biggest mistake: + +One thing to improve next week: + +--- + +# Parking Lot + +Random ideas to revisit later. + +- + +- + + +# CTO Development + +Leadership + +- + +Technology + +- + +Business + +- + +Finance + +- + +Communication + +- + +One thing I did this week that made me more like a CTO: + +- + +One thing I did this week that kept me acting like an individual contributor: + +- \ No newline at end of file From 2fc0d28bad7cc9be773188708f5596f09bab2fcd Mon Sep 17 00:00:00 2001 From: modji Date: Wed, 22 Jul 2026 10:26:32 +0200 Subject: [PATCH 6/6] vault backup: 2026-07-22 10:26:32 --- .obsidian/workspace.json | 12 +- ...tional Data & Analytics Operating Model.md | 635 ++++++++++++++++++ 2 files changed, 641 insertions(+), 6 deletions(-) create mode 100644 20 Work/Ideas/# Organizational Data & Analytics Operating Model.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index b2071fa..0b48137 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "10 Knowledge/Templates/Weekly Review Template.md", + "file": "20 Work/Ideas/# Organizational Data & Analytics Operating Model.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Weekly Review Template" + "title": "# Organizational Data & Analytics Operating Model" } } ] @@ -198,12 +198,14 @@ }, "active": "579a503273b0c5ec", "lastOpenFiles": [ - "10 Knowledge/Templates/Employe - Team Template.md", + "10 Knowledge/Templates/Idea Template.md", + "20 Work/Ideas/# Organizational Data & Analytics Operating Model.md", + "10 Knowledge/CTO Academy/Data/Data Governance.md", "10 Knowledge/Templates/Weekly Review Template.md", + "10 Knowledge/Templates/Employe - Team Template.md", "20 Work/Weekly Review", "10 Knowledge/Templates/Meeting Template.md", "10 Knowledge/Templates/Project Template.md", - "10 Knowledge/CTO Academy/Data/Data Governance.md", "10 Knowledge/CTO Academy/Data/Data Lineage.md", "10 Knowledge/CTO Academy/Data/Quizzes/Quiz 01 - Data Foundations.md", "10 Knowledge/CTO Academy/Data/Quizzes", @@ -218,7 +220,6 @@ "20 Work/Ideas/Coaching on management.md", "10 Knowledge/Templates/Journal Template.md", "10 Knowledge/Templates/Academy Template.md", - "10 Knowledge/Templates/Idea Template.md", "10 Knowledge/Templates/Decision Template.md", "10 Knowledge/Templates", "10 Knowledge/CTO Academy/Data/Data Lake vs Data Warehouse vs Lakehouse.md", @@ -228,7 +229,6 @@ "CTO Academy", "Executive Playbook.md", "CTO Academy.md", - "My Principles.md", "20 Work/Decisions", "50 Principles", "90 Archives", diff --git a/20 Work/Ideas/# Organizational Data & Analytics Operating Model.md b/20 Work/Ideas/# Organizational Data & Analytics Operating Model.md new file mode 100644 index 0000000..c4203da --- /dev/null +++ b/20 Work/Ideas/# Organizational Data & Analytics Operating Model.md @@ -0,0 +1,635 @@ + +**Date:** 2026-07-22 +**Status:** 💡 Idea + +## The Idea + +Build a company-wide **Data & Analytics capability** where trusted data is directly accessible to authorized departments, while each business unit retains ownership and expertise over its own business domain. + +The Data & Analytics team would not "own all company data." Instead, it would provide the common capability that makes organizational data: + +- Accessible + +- Trusted + +- Consistent + +- Integrated + +- Documented + +- Governed + +- Understandable + +- Actionable + + +Each department would retain its business expertise and ownership, while working closely with a dedicated member of the Data & Analytics team who understands that department, its website, projects, objectives, and data. + +The long-term goal is not simply to build more Power BI reports. + +It is to build the organization's **measurement system**, allowing departments and leadership to understand organizational performance from trusted data. + +## Problem / Opportunity + +The current organization has a relatively new data culture and historically operates through departmental silos. + +There is currently ambiguity around: + +- Who owns data + +- Who can access data + +- Who interprets data + +- Who defines KPIs + +- Who distributes information + +- Which reports are authoritative + +- Who decides whether performance is good or bad + + +Marketing has historically positioned itself as an intermediary for organizational analytics. Departments may be expected to go through Marketing to obtain or interpret their own performance data. + +This creates a risk of **information gatekeeping**. + +For example, even when centralized reports are available to departments directly, Marketing may prefer that departments request information through Marketing first. + +This model creates unnecessary dependencies: + +```text +Department + ↓ +Marketing + ↓ +Data / Reports + ↓ +Marketing interpretation + ↓ +Department +``` + +It also creates a potential conflict when data reveals uncomfortable results. + +A recent cross-platform video analytics report made previously difficult-to-see performance information directly visible. Some departments were resistant to having poor performance exposed to upper management. + +This demonstrates why business ownership should not automatically mean control over whether organizational performance information can be seen. + +Another issue became clear when the VP asked: + +> "What are the KPIs?" + +The report successfully solved the **data availability problem**, but the organization had not yet clearly established which metrics actually indicate whether video performance is succeeding or failing. + +This represents an opportunity to move beyond reporting toward true Data Governance and performance measurement. + +## Why It Matters + +A strong Data & Analytics operating model would reduce departmental silos and make trusted information directly available to the people who need it. + +Instead of asking: + +> "Who controls the numbers?" + +the organization should be able to ask: + +> "What do the numbers tell us, and what should we do about them?" + +The business value includes: + +- Better executive decision-making + +- Greater transparency + +- Less information gatekeeping + +- Consistent KPI definitions + +- Increased trust in reports + +- Faster access to information + +- More effective self-service analytics + +- Better collaboration between Data and business departments + +- Better understanding of organizational performance + +- Reduced duplication of reports and analysis + +- Less dependence on individual departments to distribute information + +- Better accountability when performance is poor + +- Cross-departmental visibility + + +The long-term objective is for leadership to understand the health of the entire organization using trusted data. + +## Who Benefits? + +- Executive leadership + +- News + +- Education + +- Marketing + +- Finance + +- HR + +- IT + +- Legal + +- Digital / Product + +- Other business departments + +- Data & Analytics team + + +Ultimately, every department should benefit from having direct access to trusted information and an analytical partner who understands its business. + +## Possible Approach + +### 1. Business Domains Own Their Business + +Departments retain ownership of their respective business domains. + +Examples: + +```text +HR → HR / Employee domain +Finance → Financial domain +Marketing → Marketing domain +News → News domain +Education → Education domain +IT → Technology / Operations domain +Legal → Legal / Compliance domain +``` + +Business ownership means departments understand their operations and participate in defining what their business metrics mean. + +It does **not** automatically mean that they control who may see organizational performance information. + +Ownership, access, reporting, interpretation, and governance are separate responsibilities. + +### 2. Data & Analytics Provides the Shared Capability + +The Data & Analytics function should be responsible for establishing and maintaining the organization's shared analytics capability. + +Responsibilities could include: + +- Data integration + +- Analytics architecture + +- Power BI / reporting + +- Data quality + +- Data lineage + +- Metadata + +- KPI documentation + +- Shared definitions + +- Cross-domain analytics + +- Analytical methodology + +- Self-service analytics + +- Data access implementation + +- Data governance standards + + +The objective is not: + +> "All data belongs to Data & Analytics." + +Instead: + +> **Business units own their business domains. Data & Analytics makes organizational data trustworthy, integrated, consistently measured, understandable, and accessible to authorized users.** + +### 3. Dedicated Data / Technology Partners + +Each department already has a dedicated team member who works closely with it on its website, projects, technology, and evolution. + +This relationship should evolve into an **embedded Data / Technology partnership**. + +```text + Data & Analytics + │ + ┌──────────────┼──────────────┐ + │ │ │ + Team Member Team Member Team Member + ↕ ↕ ↕ + News Education Other Dept. +``` + +These team members should develop enough domain knowledge to understand and interpret the department's analytics. + +They should not merely provide numbers. + +They should be able to: + +- Explain what happened + +- Identify patterns + +- Investigate changes + +- Challenge assumptions + +- Understand the department's digital environment + +- Connect technical events with analytics + +- Recommend further investigation + +- Work with departmental experts to understand why something happened + + +The business department still retains accountability for business decisions. + +### 4. Shared Interpretation + +Interpretation should not automatically belong to Marketing. + +Instead: + +**Data & Analytics asks:** + +> What happened? + +> Where did it happen? + +> What patterns exist? + +> Is the data reliable? + +**Data + Department together ask:** + +> Why did it happen? + +The department contributes business context because it understands its operations. + +The embedded Data/Technology partner contributes analytical and technical expertise because they understand both the data and the department. + +**Department / Leadership decides:** + +> Is this performance acceptable? + +> What should we do about it? + +Marketing can contribute Marketing expertise where relevant, but should not automatically become the interpretation layer for News, Education, or other departments. + +### 5. Self-Service by Default + +Authorized users should not need to request basic information from Marketing or Data if trusted reports already exist. + +The desired model is: + +```text + Trusted Data & Analytics + │ + ┌─────────────┼─────────────┐ + ↓ ↓ ↓ + News Education Marketing + ↕ ↕ ↕ + Data Partner Data Partner Data Partner +``` + +Not: + +```text +Department → Marketing → Data → Marketing → Department +``` + +The goal is not to replace Marketing as the gatekeeper with Data as the new gatekeeper. + +The goal is to **remove unnecessary gatekeeping entirely**. + +### 6. Establish KPI Governance + +Reports should distinguish between: + +**Metrics** + +Numbers that describe what happened. + +and: + +**KPIs** + +Metrics specifically selected to determine whether the organization is succeeding against an objective. + +For example, a video analytics report may contain dozens of metrics. + +Leadership may ultimately decide that only a small number represent organizational success. + +The process should become: + +```text +Business Objective + ↓ +Agreed KPI + ↓ +Business Definition + ↓ +Documented Calculation + ↓ +Trusted Data Source + ↓ +Power BI / Analytics + ↓ +Department Interpretation + ↓ +Leadership Decision +``` + +Data & Analytics should facilitate this process but should not independently invent corporate KPIs. + +Questions to establish for important KPIs: + +- What business objective does this KPI measure? + +- Who owns its business definition? + +- How exactly is it calculated? + +- Which systems contribute data? + +- What are its limitations? + +- How frequently is it refreshed? + +- Who may access it? + +- Who approves changes? + +- Where is the authoritative version? + +- What supporting metrics explain changes? + + +### 7. Position Marketing's Data Analyst Correctly + +Marketing having a Data Analyst is not inherently a problem. + +The important question is the analyst's mandate. + +A Marketing Data Analyst could appropriately specialize in: + +- Marketing campaigns + +- Acquisition + +- Audience segmentation + +- Marketing effectiveness + +- Advertising performance + +- Marketing KPIs + + +The concern would be if the role evolves into: + +> The organizational analyst responsible for interpreting News, Education, Digital, Streaming, and other departments. + +Marketing should be one business domain within the broader Data & Analytics ecosystem, not the mandatory gateway to organizational information. + +Do not make the organizational argument: + +> "Marketing shouldn't have an analyst." + +Instead establish the broader model in which that analyst naturally becomes a **Marketing domain specialist**. + +### 8. Expand Beyond Audience Analytics + +The current video analytics work can become the foundation rather than the final destination. + +Long term, gradually integrate authorized information from additional domains. + +```text + ORGANIZATIONAL HEALTH + + Financial Audience People Operations + │ │ │ │ + Revenue Reach Headcount SLA + Budget Viewing Turnover Uptime + Costs Retention Hiring Delivery + │ │ │ │ + └──────────────┴──────┬──────┴──────────────┘ + ↓ + Executive Decisions +``` + +Potential domains include: + +**Finance** + +- Revenue + +- Budget + +- Actual vs forecast + +- Costs + + +**HR** + +- Headcount + +- Hiring + +- Turnover + +- Workforce trends + + +**IT** + +- SLA + +- Incidents + +- Uptime + +- Project delivery + +- Service requests + + +**Legal** + +- Compliance indicators + +- Contracts + +- Rights / licensing where applicable + + +**Audience / Content** + +- Reach + +- Consumption + +- Engagement + +- Retention + +- Content performance + + +The goal is not one enormous Power BI dashboard. + +The goal is an interconnected **organizational data ecosystem**. + +### 9. Respect Sensitive Data Ownership + +Expanding into HR, Finance and Legal should not mean requesting unrestricted access to everything. + +These departments may legitimately need strict controls around sensitive information. + +Instead: + +> The department retains ownership and appropriate access control while participating in the organization's governed analytics ecosystem. + +For example, HR may expose approved workforce metrics without exposing individual salaries or unnecessary personal information. + +This reduces resistance and avoids presenting Data & Analytics as an attempt to take control away from departments. + +## Effort + +**Estimated complexity:** High + +This is not primarily a Power BI or technical project. + +It requires: + +- Organizational change + +- Executive sponsorship + +- Data Governance + +- Departmental cooperation + +- Clear responsibilities + +- Trust + +- Political navigation + +- Technical architecture + +- Gradual cultural change + + +Implementation should therefore be incremental rather than presented as one large transformation project. + +## Risks / Questions + +- Marketing currently has strong relationships with the VP and CEO. + +- Marketing has already received approval to hire a Data Analyst focused on interpretation without consultation with the existing Data & Analytics function. + +- Marketing may perceive direct self-service analytics as reducing its organizational influence. + +- Departments may resist transparency when data reveals poor performance. + +- HR and Finance may resist participation because of concerns around control and sensitive information. + +- Departments may interpret "central Data & Analytics" as an attempt to take ownership of their data. + +- KPI ownership is currently unclear. + +- Leadership may not yet recognize the distinction between metrics and KPIs. + +- There may be no formal Data Governance mandate. + +- Data access and business ownership may currently be confused. + +- Embedded team members need sufficient analytical and business knowledge to become credible partners. + +- Who ultimately resolves disagreements over KPI definitions? + +- Who should sponsor Data Governance at the executive level? + +- How should cross-department KPIs be owned? + +- What should the formal mandate of Data & Analytics become? + + +## Next Step + +Use the existing **cross-platform video analytics report** as the first practical example of this operating model. + +Do not initially attempt to restructure the organization or challenge Marketing's analyst position. + +Instead, follow up on the VP's question: + +> **"What are the KPIs?"** + +Work with leadership and the relevant departments to identify: + +1. What are our business objectives for video? + +2. Which metrics indicate whether we are succeeding? + +3. Who owns the business definition of each KPI? + +4. How should each KPI be calculated? + +5. Which supporting metrics help explain performance? + + +Document the agreed definitions and implement them consistently in the existing report. + +This creates a small, concrete example of Data Governance and demonstrates the value of the broader operating model. + +From there, gradually extend the model to additional domains rather than trying to win the abstract argument over **"Who is Data?"** + +The long-term strategy is: + +> **Don't fight to be called Data. Build the organizational Data & Analytics capability until its role becomes self-evident.** + +## Related + +- [[Data Governance]] + +- [[Data Quality]] + +- [[Data Lineage]] + +- [[Semantic Layer]] + +- [[Business Glossary]] + +- [[KPI]] + +- [[Data Owner]] + +- [[Data Steward]] + +- [[Power BI]] + +- [[Data Culture]] + +- [[Self-Service Analytics]] + +- [[CTO]] \ No newline at end of file