import numpy as np
import pandas as pd
SEED = 2026 # fixed, so every number in the post reproduces
TAKEOVERS = 240 # announced takeovers, as in part 1
LEAKS = 48 # 1 in 5 leak, Patel and Putnins (2020)
ORDINARY_BUYERS = 300 # clients who buy each target for other reasons
# Ordinary purchases: the three quartiles that Cheng and co-authors report for all their
# transaction cycles (2023 version, Table 4, Panel B, training set), with a lowest and a
# highest value that are assumptions.
DAYS_BEFORE = [1, 2, 7, 18, 40] # trading days before the announcement
PORTFOLIO_SHARE = [0, 0.0879, 0.2188, 0.5019, 1]
SIZE_VS_LARGEST = [0, 0.0745, 0.1949, 0.4374, 2]
LOG_ACCOUNT_AGE = [np.log(30), 5.9839, 7.5464, 8.1476, np.log(10_000)]
HOME_CITY_SHARE = 0.072 # 7.2% of all their transaction cycles
def draw_from_quartiles(rng, values, n):
"""n draws with the lowest value, three quartiles and highest value in values. rng.uniform
draws a number u between 0 and 1 with equal chance, and np.interp reads the value at u off
the straight lines through the points (0, lowest), (0.25, first quartile), (0.5, median),
(0.75, third quartile) and (1, highest), so a quarter of the draws fall between each pair."""
return np.interp(rng.uniform(size=n), [0, 0.25, 0.5, 0.75, 1], values)
def ordinary_purchases(rng, n):
"""The five features of n purchases made for ordinary reasons."""
return pd.DataFrame({
"days_before": np.round(draw_from_quartiles(rng, DAYS_BEFORE, n)),
"portfolio_share": draw_from_quartiles(rng, PORTFOLIO_SHARE, n),
"size_vs_largest": draw_from_quartiles(rng, SIZE_VS_LARGEST, n),
"account_age_days": np.round(np.exp(draw_from_quartiles(rng, LOG_ACCOUNT_AGE, n))),
"home_city": rng.uniform(size=n) < HOME_CITY_SHARE,
})
def simulate_purchases(seed):
"""One row per purchase: a client who bought a takeover target in the 40 trading days
before the announcement. The column insider is the answer key, used only to check flags."""
# rng is the random number generator, and the same seed always gives the same draws.
rng = np.random.default_rng(seed)
# The insiders' first purchase: np.arange(1, 41) gives 1 to 40 trading days before the
# announcement, stopping before 41, and np.where gives each of the last 6 days a
# probability of 0.5 / 6 and each of the 34 days before them 0.5 / 34.
distances = np.arange(1, 41)
weights = np.where(distances <= 6, 0.5 / 6, 0.5 / 34)
tables = []
for takeover in range(TAKEOVERS):
# assign() adds columns with one value for every row: here the takeover's number and
# insider=False for the ordinary buyers.
tables.append(ordinary_purchases(rng, ORDINARY_BUYERS)
.assign(takeover=takeover, insider=False))
if takeover < LEAKS:
# 1 to 3 insider accounts, 2 on average, close to the 2.16 accounts per case in the
# CSRC cases; that all of them are clients of this broker is an assumption.
# rng.integers(1, 4) draws a whole number from 1 to 3, because it stops before 4.
n = rng.integers(1, 4)
insiders = ordinary_purchases(rng, n)
# rng.choice(distances, n, p=weights) draws n first-purchase days with those
# probabilities, as the days of insider trading in part 1 (Meulbroek 1992).
insiders["days_before"] = rng.choice(distances, n, p=weights)
# Of the prosecuted cases Cheng and co-authors read, 38.0% mention an unbalanced
# portfolio, 53.4% an insider who invested more than their history, 9.7% an account
# set up just for the insider trading, and 61.5% an insider in the firm's home city.
# With those chances, an insider's ordinary draw is replaced: 80% to 100% of the
# portfolio in the target, 1 to 10 times the largest earlier position, an account
# opened 1 to 60 days before the first purchase (assumed ranges), and a
# home-city link. .loc[mask, column] = values replaces the values in the rows where
# mask is True.
unbalanced = rng.uniform(size=n) < 0.380
insiders.loc[unbalanced, "portfolio_share"] = rng.uniform(0.8, 1, unbalanced.sum())
larger = rng.uniform(size=n) < 0.534
insiders.loc[larger, "size_vs_largest"] = rng.uniform(1, 10, larger.sum())
new_account = rng.uniform(size=n) < 0.097
insiders.loc[new_account, "account_age_days"] = np.round(
rng.uniform(1, 60, new_account.sum()))
insiders["home_city"] = rng.uniform(size=n) < 0.615
tables.append(insiders.assign(takeover=takeover, insider=True))
# pd.concat stacks the tables, and ignore_index=True numbers the rows 0, 1, 2 and so on.
return pd.concat(tables, ignore_index=True)
def make_like_ordinary(purchases, seed):
"""The same purchases, with each insider's features other than timing drawn like an
ordinary purchase's: an old account, an ordinary position, and usually no home-city link."""
# A second generator, started from seed + 1, so the ordinary buyers and the insiders' timing
# stay exactly as they are. copy() makes like_ordinary a separate table.
rng = np.random.default_rng(seed + 1)
like_ordinary = purchases.copy()
rows = like_ordinary["insider"]
redrawn = ordinary_purchases(rng, rows.sum())
# to_numpy() copies the redrawn values by position: assigning the column itself would match
# rows by their labels, and the redrawn table is numbered from 0.
for column in ["portfolio_share", "size_vs_largest", "account_age_days", "home_city"]:
like_ordinary.loc[rows, column] = redrawn[column].to_numpy()
return like_ordinary
purchases = simulate_purchases(SEED)Detecting unusual trading with an isolation forest, part 2: broker records
Part 1 of this pair looks for illegal insider trading in each stock’s daily return and volume with an isolation forest, which singles out the observations that are easiest to separate from the rest. There, most days of insider trading look like ordinary days. With market data, I only see transactions and quotes, with no ID and no portfolio. A broker can connect each transaction to a client, the client’s other transactions and the whole portfolio. It may see, for example, that a client sold everything and made one single bet on one stock, which then went up by 200%.
After a takeover is announced, you can look back in a broker’s records at the clients who bought the target in the weeks before. Hundreds bought it for ordinary reasons, and a few may be illegal insiders, clients who traded on the offer while it was inside information.
In the EU, Article 16 of the Market Abuse Regulation requires a broker to detect and report suspicious orders and transactions.
I simulate the buyers of 240 takeover targets at one broker, with the offer leaking in 1 in 5 takeovers, as in Part 1, and an isolation forest flags three of each target’s roughly 300 buyers. Choosing three buyers at random would flag 1% of the illegal insiders’ purchases. When illegal insiders trade like those in prosecuted cases, the forest flags 36.3% of their purchases, mostly from an account in the target’s home city, a link few ordinary buyers have. When they buy like ordinary buyers, it flags 1.1%, no better than chance.
Simulating the data
The simulation follows Cheng and co-authors, who screen for insider trading with an isolation forest at one of China’s largest brokerages. They score 652,307 transaction cycles, one investor’s position in one stock around an event such as a takeover, and report the quartiles of five features. They also report how often each trait of the illegal insiders appears in the cases prosecuted by China’s securities regulator, the CSRC. Like them, I have a set of variables on which the ordinary buyers look ordinary, while the illegal insiders deviate on some of them. Some of the values are my own, arbitrary choices.
Each row is one client’s purchase of one target in the 40 trading days before the announcement. The forest never sees which buyers are illegal insiders, and I only use that to check which flags fall on them.
- Ordinary buyers. 300 clients buy each of the 240 targets, and the five features of their purchases, defined below, follow those quartiles.
- Illegal insiders. The offer leaks in 48 takeovers, 1 in 5 as in Part 1, each with 1 to 3 illegal insider accounts at the broker, 2 on average, close to the 2.16 accounts per CSRC case. An illegal insider’s first purchase follows the insider-trading timing described in Part 1, from prosecuted cases.
- Their traits. Living close to the company is one example of how illegal insiders stand out. In the CSRC cases, 61.5% live in the city of the target’s headquarters, against 7.2% of all transaction cycles, 53.4% make a purchase larger than any earlier position, 38.0% have an unbalanced portfolio, and 9.7% open an account only for the trade. Each illegal insider gets each trait with that chance.
I match each ordinary feature to the quartiles in the Cheng and co-authors paper, with lowest and highest values that I choose arbitrarily. I draw a number \(u\) between 0 and 1, all values equally likely, and read the feature \(x\) off straight lines through five points:
\[x = q_k + \frac{u - p_k}{p_{k+1} - p_k}\,(q_{k+1} - q_k) \qquad \text{for } p_k \le u \le p_{k+1},\]
where \(p = (0, 0.25, 0.5, 0.75, 1)\) and \(q\) holds the lowest value, the three quartiles and the highest value:
| Feature | Lowest | First quartile | Median | Third quartile | Highest |
|---|---|---|---|---|---|
| days before the announcement | 1 | 2 | 7 | 18 | 40 |
| portfolio share | 0 | 0.0879 | 0.2188 | 0.5019 | 1 |
| size against the largest earlier position | 0 | 0.0745 | 0.1949 | 0.4374 | 2 |
| account age, logarithm of days | log 30 | 5.9839 | 7.5464 | 8.1476 | log 10,000 |
A home-city link has a chance of 7.2%, and I draw each feature independently of the others.
A purchase by an illegal insider starts from the same draws, and then:
- Timing. The first purchase is 1 to 6 trading days before the announcement with a 1/2 chance, and 7 to 40 days before it otherwise, like the insider-trading days in Part 1.
- Traits. With a chance of 38.0%, the portfolio share is drawn again between 0.8 and 1. With a chance of 53.4%, the size against the largest earlier position is drawn again between 1 and 10. With a chance of 9.7%, the account is opened 1 to 60 days before the first purchase. The home-city link has a chance of 61.5%.
Computing five features
From the broker’s records up to the day before the announcement, I use five features: the number of trading days before the announcement that the client first buys, the target’s share of the client’s portfolio, the size of the position against the client’s largest earlier position, the age of the account, and whether the client’s account is at a branch in the target’s home city, a stand-in for living there. I use these five to replicate Cheng and co-authors. In market surveillance, you could probably use many more, and the isolation forest runs the same way however many features you include.
Flagging three purchases per takeover
The isolation forest uses these features and tries to isolate each purchase. Think of one variable with the numbers 1, 2, 3, 4, 5 and 100. No matter how many random splits I draw, 100 usually needs the fewest splits to be isolated. The forest does the same with more variables, which is harder to picture but works the same way. It splits a random sample of 256 purchases until each is alone, 100 times, and a purchase that takes few splits gets a high score.
I fit the forest once on all 240 takeovers, pooled as Cheng and co-authors do, and flag the three purchases with the highest score among each target’s buyers, 720 flags in all. These purchases need closer review.
from sklearn.ensemble import IsolationForest
INPUTS = ["days_before", "portfolio_share", "size_vs_largest", "log_account_age", "home_city"]
FLAGS_PER_TAKEOVER = 3 # 1% of about 300 buyers
def flag_purchases(purchases):
"""Draw the forest's random splits from all purchases, score them, and flag the three
with the highest isolation score among each target's buyers.
Account age spans days to decades, so the forest receives its logarithm, which gives equal
steps to 30, 300 and 3,000 days. scikit-learn's defaults repeat the random splitting 100
times (n_estimators=100), each time on 256 purchases drawn at random (max_samples="auto"),
so neither is written out. random_state fixes the random draws, so every run gives the same
scores.
score_samples() returns minus the isolation score, so a minus sign gives the score itself.
groupby("takeover") ranks each target's buyers separately: rank(ascending=False) gives 1 to
the highest score, and method="first" breaks ties by row order, so ranks 1 to 3 mark
exactly three purchases per takeover.
"""
flagged = purchases.assign(log_account_age=np.log(purchases["account_age_days"]))
forest = IsolationForest(random_state=SEED).fit(flagged[INPUTS])
flagged["isolation_score"] = -forest.score_samples(flagged[INPUTS])
rank = flagged.groupby("takeover")["isolation_score"].rank(ascending=False, method="first")
flagged["flagged"] = rank <= FLAGS_PER_TAKEOVER
return flagged
purchases = flag_purchases(purchases)
print(f"purchases: {len(purchases):,}, of which insider purchases: {purchases['insider'].sum()}, "
f"flagged: {purchases['flagged'].sum()}")purchases: 72,091, of which insider purchases: 91, flagged: 720
Counting the flagged purchases by illegal insiders
Patel and Putniņš find that insider trading is more likely to be detected when it is unusual, so the illegal insiders who were never caught may buy more like ordinary buyers. So I also flag a second version, where the illegal insiders buy on the same days as before, but their position, portfolio share, account age and home city are drawn like an ordinary buyer’s. For each version, I count the flagged purchases.
def purchase_kind(purchases):
"""Each purchase's kind, insider or ordinary, as a Series of names, one per row."""
return pd.Series(np.where(purchases["insider"], "insider purchases", "ordinary purchases"),
index=purchases.index)
def flag_counts(purchases, groups, decimals=1):
"""Purchases, flagged and not flagged, and the percentage flagged, for each group of
purchases. agg() counts each group: size is the number of purchases, sum the number
flagged, since True counts as 1, and mean the share flagged."""
table = purchases.groupby(groups)["flagged"].agg(["size", "sum", "mean"])
table.columns = ["purchases", "flagged", "% flagged"]
table.insert(2, "not flagged", table["purchases"] - table["flagged"])
table["% flagged"] = (100 * table["% flagged"]).round(decimals)
return table
def leaks_flagged(purchases):
"""The leaked takeovers with at least one flagged insider purchase, and all leaked
takeovers: any() is True where at least one of a leak's insider purchases is flagged."""
by_leak = purchases.loc[purchases["insider"]].groupby("takeover")["flagged"].any()
return by_leak.sum(), len(by_leak)
like_ordinary = flag_purchases(make_like_ordinary(purchases, SEED))
# "{:,}".format writes a count with thousands separators, such as 72,000, and col_space=12
# gives every column at least 12 characters, so the headings stand apart
with_commas = {column: "{:,}".format for column in ["purchases", "flagged", "not flagged"]}
for name, table in [("insiders like prosecuted cases", purchases),
("insiders like ordinary buyers", like_ordinary)]:
flagged_leaks, leaks = leaks_flagged(table)
print(name)
print(flag_counts(table, purchase_kind(table)).to_string(formatters=with_commas,
col_space=12))
print(f"leaked takeovers with an insider purchase flagged: {flagged_leaks} of {leaks} "
f"({100 * flagged_leaks / leaks:.1f}%)\n")
# Choosing three of a target's buyers at random flags each of them with a chance of 3 divided by
# the number of the target's buyers, so the mean of that chance over the insider purchases is the
# share of them that a random choice would flag. transform("size") gives every purchase the
# number of purchases of its takeover.
buyers = purchases.groupby("takeover")["takeover"].transform("size")
random_pct = 100 * (FLAGS_PER_TAKEOVER / buyers[purchases["insider"]]).mean()
print(f"choosing 3 buyers per takeover at random would flag {random_pct:.1f}% of the insider "
"purchases")insiders like prosecuted cases
purchases flagged not flagged % flagged
insider purchases 91 33 58 36.3
ordinary purchases 72,000 687 71,313 1.0
leaked takeovers with an insider purchase flagged: 28 of 48 (58.3%)
insiders like ordinary buyers
purchases flagged not flagged % flagged
insider purchases 91 1 90 1.1
ordinary purchases 72,000 719 71,281 1.0
leaked takeovers with an insider purchase flagged: 1 of 48 (2.1%)
choosing 3 buyers per takeover at random would flag 1.0% of the insider purchases
When the illegal insiders trade like the prosecuted cases, the forest flags 33 of their 91 purchases, and at least one of them in 28 of the 48 leaked takeovers. But 687 of its 720 flags, 95.4%, fall on ordinary buyers. When the illegal insiders buy like ordinary buyers, it flags only 1 of their 91 purchases, in 1 of the 48 leaked takeovers.
Why these illegal insiders stand out
# Adding the two names gives each group a label, such as "insider purchases with a home-city
# link". The percentages keep two decimals, because some are below 0.1.
link = np.where(purchases["home_city"], " with a home-city link", " without a home-city link")
print(flag_counts(purchases, purchase_kind(purchases) + link, decimals=2)
.to_string(formatters=with_commas, col_space=12)) purchases flagged not flagged % flagged
insider purchases with a home-city link 56 32 24 57.14
insider purchases without a home-city link 35 1 34 2.86
ordinary purchases with a home-city link 5,224 661 4,563 12.65
ordinary purchases without a home-city link 66,776 26 66,750 0.04
32 of the 33 flagged purchases by illegal insiders have a home-city link. The link alone is not enough, since the forest flags only 12.65% of the ordinary purchases with the link, so the illegal insiders also stand out on size, portfolio share or account age.
import matplotlib.pyplot as plt
TEAL, GREY = "#17868A", "#888888"
# The site's chart colours, in order: the cream background, the text, the light grid lines,
# the pale frame and the axis labels, the same on every chart of the site.
BG, INK, GRID, SPINE, AXIS_TEXT = "#FCEFE3", "#1f1f1f", "#EADCCC", "#D5C6B4", "#4a4a4a"
# The share of the insider purchases flagged, for each version of the insiders and for three
# buyers per takeover chosen at random
shares = [100 * purchases.loc[purchases["insider"], "flagged"].mean(),
100 * like_ordinary.loc[like_ordinary["insider"], "flagged"].mean(),
random_pct]
names = ["Insiders like\nprosecuted cases", "Insiders like\nordinary buyers",
"3 buyers per takeover\nchosen at random"]
# figsize is the figure's width and height in inches
figure, axis = plt.subplots(figsize=(7, 4.2))
bars = axis.bar(names, shares, color=[TEAL, TEAL, GREY], width=0.55)
# bar_label writes each bar's value above it, padding=4 points away
axis.bar_label(bars, labels=[f"{share:.1f}%" for share in shares], padding=4, color=INK)
axis.set_ylabel(f"% of the {purchases['insider'].sum()} insider purchases flagged")
axis.set_ylim(0, 42)
# the site's cream background, light grid and pale frame
figure.patch.set_facecolor(BG)
axis.set_facecolor(BG)
axis.grid(True, axis="y", color=GRID, lw=1.0)
axis.set_axisbelow(True)
for side in ("top", "right"):
axis.spines[side].set_visible(False)
for side in ("left", "bottom"):
axis.spines[side].set_color(SPINE)
axis.tick_params(axis="both", length=0, colors=INK, pad=6)
axis.yaxis.label.set_color(AXIS_TEXT)
# tight_layout() fits the labels inside the figure. savefig writes the chart to a PNG file:
# dpi=140 sets its resolution, bbox_inches="tight" trims the empty margin, and facecolor=BG
# keeps the cream background. show() displays the chart.
plt.tight_layout()
plt.savefig("flags_insider_share.png", dpi=140, bbox_inches="tight", facecolor=BG)
plt.show()
In the chart, the teal bars show the forest’s shares of purchases by illegal insiders flagged, and the gray bar shows what three randomly selected buyers per takeover would flag.
Limitations
The isolation forest flags unusual behavior, but unusual behavior is not the same as a crime. A flagged buyer may have been lucky.
The forest finds what stands out. I assume illegal insiders stand out because their trading on the information moves the price, for example, in a run-up before the announcement, or causes abnormal volume, and a broker sees even more than market data. But if the illegal insiders trade in the same pattern as everyone else, the method does not work, as the second version shows.
Conclusion
So a broker can connect the dots that market data cannot, and the isolation forest flags the buyers who stand out for a closer review. It only finds the illegal insiders who trade differently from everyone else.
Read next
- Detecting unusual trading with an isolation forest, part 1: market data What an isolation forest flags in daily market data, and why it flags few days of insider trading.
- Estimating the frequency of market crimes What you can estimate about market crime when most crimes are never detected.
Disclaimer: a teaching example on simulated data. Not investment advice.
Sources: Guang Cheng, Christian T. Lundblad, Zhishu Yang and Qi Zhang, Detecting Insider Trading in the Era of Big Data and Machine Learning, SSRN working paper, 2023 version (the link opens the latest version), for the transaction cycles, the features, the quartiles of all transaction cycles and the traits of prosecuted insiders. Lisa K. Meulbroek, An Empirical Analysis of Illegal Insider Trading, Journal of Finance (1992), for the timing of insider trading. Vinay Patel and Tālis J. Putniņš, How Much Insider Trading Happens in Stock Markets?, SSRN working paper (2020), for the share of takeovers that leak and for detection being more likely when trading is unusual. Regulation (EU) No 596/2014 on market abuse, Article 16, for the duty to detect and report suspicious orders and transactions. scikit-learn, IsolationForest, for the method.