Social Timeline & Newsfeed (Twitter/X & Meta Feed)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

-- PostgreSQL / CockroachDB Sharded Authoritative Tweet Storage
CREATE TABLE tweets (
    tweet_id BIGINT PRIMARY KEY,              -- 64-bit Snowflake ID (timestamp encoded)
    author_id BIGINT NOT NULL,                -- User ID of author
    content VARCHAR(280) NOT NULL,            -- UTF-8 text
    media_urls JSONB NULL,                    -- Array of S3 / CDN media asset links
    reply_to_tweet_id BIGINT NULL,            -- Parent tweet ID if thread
    retweet_count INT NOT NULL DEFAULT 0,
    like_count INT NOT NULL DEFAULT 0,
    created_at TIMESTAMP WITH TIME ZONE NOT NULL
);

-- Sharded by author_id to colocate an author's tweets on a single storage node
CREATE INDEX idx_tweets_author_created ON tweets (author_id, created_at DESC);

-- Social Graph Relationship Table (TAO Edge Model / PostgreSQL)
CREATE TABLE user_follows (
    follower_id BIGINT NOT NULL,              -- The user who is following
    followee_id BIGINT NOT NULL,              -- The user being followed
    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
    PRIMARY KEY (followee_id, follower_id)    -- Composite PK optimized for "get_followers"
);

CREATE INDEX idx_user_follows_reverse ON user_follows (follower_id, followee_id);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

// Production Hybrid Timeline Aggregator & K-Way Merge
export interface TweetSummary {
  tweetId: bigint;
  authorId: bigint;
  timestamp: number;
}

const CELEBRITY_FOLLOWER_THRESHOLD = 25000;

export class TimelineService {
  // FANOUT WORKER WRITE PATH
  async onTweetPublished(tweet: TweetSummary, followers: bigint[]): Promise<void> {
    // Always write to author's own profile timeline
    await redis.zadd(`user_timeline:${tweet.authorId}`, tweet.timestamp, tweet.tweetId.toString());

    if (followers.length >= CELEBRITY_FOLLOWER_THRESHOLD) {
      // CELEBRITY: Bypass push fanout! Zero follower updates written.
      console.log(`Celebrity post detected for user ${tweet.authorId}. Bypassing push fanout.`);
      return;
    }

    // NORMAL USER: Push tweet ID into every follower's home timeline ZSET in batches
    const pipeline = redis.pipeline();
    for (const followerId of followers) {
      const key = `home_timeline:${followerId}`;
      pipeline.zadd(key, tweet.timestamp, tweet.tweetId.toString());
      pipeline.zremrangebyrank(key, 0, -801); // Retain top 800 items
    }
    await pipeline.exec();
  }

  // TIMELINE READ PATH
  async getHomeTimeline(
    userId: bigint,
    followedCelebrities: bigint[],
    limit: number = 20,
    maxId?: bigint
  ): Promise<string[]> {
    // 1. Fetch normal pre-computed timeline from Redis ZSET
    const normalTweets = await redis.zrevrangebyscore(
      `home_timeline:${userId}`,
      maxId ? (maxId - 1n).toString() : '+inf',
      '-inf',
      'LIMIT', 0, limit
    );

    // 2. Concurrently fetch top tweets from all followed celebrities
    const celebrityPipelines = redis.pipeline();
    for (const celebId of followedCelebrities) {
      celebrityPipelines.zrevrangebyscore(
        `user_timeline:${celebId}`,
        maxId ? (maxId - 1n).toString() : '+inf',
        '-inf',
        'LIMIT', 0, limit
      );
    }
    const celebrityResults = await celebrityPipelines.exec();

    // 3. In-Memory K-Way Merge Sort across normal timeline and celebrity feeds
    const allCandidateIds: bigint[] = normalTweets.map((id) => BigInt(id));
    for (const [, tweetList] of celebrityResults ?? []) {
      if (Array.isArray(tweetList)) {
        for (const id of tweetList) allCandidateIds.push(BigInt(id));
      }
    }

    // Sort by the course's chosen cursor key. Snowflake IDs are roughly time ordered,
    // but cross-worker clock skew means they are not a perfect global chronology.
    allCandidateIds.sort((a, b) => (b > a ? 1 : b < a ? -1 : 0));

    // Return the top N requested tweet IDs for hydration
    return allCandidateIds.slice(0, limit).map((id) => id.toString());
  }
}